From 81fa31f50b045174258aabb1b3a474096757c31d Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Fri, 29 Nov 2024 16:55:33 +0100 Subject: [PATCH] chore: add basic tests for S3 storage --- pyproject.toml | 1 + .../0025_alter_datalayer_geojson.py | 24 ++++ umap/models.py | 10 +- umap/storage.py | 6 +- umap/tests/test_datalayer_s3.py | 135 ++++++++++++++++++ 5 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 umap/migrations/0025_alter_datalayer_geojson.py create mode 100644 umap/tests/test_datalayer_s3.py diff --git a/pyproject.toml b/pyproject.toml index 8951cab6..20c0cc63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ test = [ "pytest-playwright==0.6.2", "pytest-rerunfailures==15.0", "pytest-xdist>=3.5.0,<4", + "moto[s3]==5.0.21" ] docker = [ "uwsgi==2.0.28", diff --git a/umap/migrations/0025_alter_datalayer_geojson.py b/umap/migrations/0025_alter_datalayer_geojson.py new file mode 100644 index 00000000..0dd9d7c0 --- /dev/null +++ b/umap/migrations/0025_alter_datalayer_geojson.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.2 on 2024-11-29 15:45 + +from django.db import migrations, models + +import umap.models + + +class Migration(migrations.Migration): + dependencies = [ + ("umap", "0024_alter_map_share_status"), + ] + + operations = [ + migrations.AlterField( + model_name="datalayer", + name="geojson", + field=models.FileField( + blank=True, + null=True, + storage=umap.models.set_storage, + upload_to=umap.models.upload_to, + ), + ), + ] diff --git a/umap/models.py b/umap/models.py index de7174a7..900c6c8d 100644 --- a/umap/models.py +++ b/umap/models.py @@ -426,7 +426,7 @@ class Pictogram(NamedModel): attribution = models.CharField(max_length=300) category = models.CharField(max_length=300, null=True, blank=True) - pictogram = models.FileField(upload_to="pictogram", storage=storages["default"]) + pictogram = models.FileField(upload_to="pictogram") @property def json(self): @@ -445,6 +445,10 @@ def upload_to(instance, filename): return instance.geojson.storage.make_filename(instance) +def set_storage(): + return storages["data"] + + class DataLayer(NamedModel): """ Layer to store Features in. @@ -470,7 +474,7 @@ class DataLayer(NamedModel): map = models.ForeignKey(Map, on_delete=models.CASCADE) description = models.TextField(blank=True, null=True, verbose_name=_("description")) geojson = models.FileField( - upload_to=upload_to, blank=True, null=True, storage=storages["data"] + upload_to=upload_to, blank=True, null=True, storage=set_storage ) display_on_load = models.BooleanField( default=False, @@ -519,7 +523,7 @@ class DataLayer(NamedModel): new.pk = uuid.uuid4() if map_inst: new.map = map_inst - new.geojson = File(new.geojson.file.file) + new.geojson = File(new.geojson.file.file, name="tmpname") new.save() return new diff --git a/umap/storage.py b/umap/storage.py index 684e90d3..c03d4ccb 100644 --- a/umap/storage.py +++ b/umap/storage.py @@ -7,6 +7,7 @@ from pathlib import Path from botocore.exceptions import ClientError from django.conf import settings from django.contrib.staticfiles.storage import ManifestStaticFilesStorage +from django.core.files.base import File from django.core.files.storage import FileSystemStorage from rcssmin import cssmin from rjsmin import jsmin @@ -112,7 +113,10 @@ class UmapS3(S3Storage): pass def onDatalayerDelete(self, instance): - pass + return self.connection.meta.client.delete_object( + Bucket=self.bucket_name, + Key=instance.geojson.name, + ) class UmapFileSystem(FileSystemStorage): diff --git a/umap/tests/test_datalayer_s3.py b/umap/tests/test_datalayer_s3.py new file mode 100644 index 00000000..bcdee4a8 --- /dev/null +++ b/umap/tests/test_datalayer_s3.py @@ -0,0 +1,135 @@ +import json +import os + +import boto3 +import pytest +from botocore.errorfactory import ClientError +from django.core.files.base import ContentFile +from django.core.files.storage import storages +from moto import mock_aws + +from umap.models import DataLayer + +from .base import DataLayerFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture(scope="module", autouse=True) +def patch_storage(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + before = DataLayer.geojson.field.storage + + DataLayer.geojson.field.storage = storages.create_storage( + { + "BACKEND": "umap.storage.UmapS3", + "OPTIONS": { + "access_key": "testing", + "secret_key": "testing", + "bucket_name": "umap", + "region_name": "us-east-1", + }, + } + ) + yield + DataLayer.geojson.field.storage = before + + +@pytest.fixture(scope="module", autouse=True) +def mocked_aws(): + """ + Mock all AWS interactions + Requires you to create your own boto3 clients + """ + with mock_aws(): + client = boto3.client("s3", region_name="us-east-1") + client.create_bucket(Bucket="umap") + client.put_bucket_versioning( + Bucket="umap", VersioningConfiguration={"Status": "Enabled"} + ) + yield + + +def test_can_create_datalayer(map, datalayer): + other = DataLayerFactory(map=map) + assert datalayer.geojson.name == f"{datalayer.pk}.geojson" + assert other.geojson.name == f"{other.pk}.geojson" + + +def test_clone_should_return_new_instance(map, datalayer): + clone = datalayer.clone() + assert datalayer.pk != clone.pk + assert datalayer.name == clone.name + assert datalayer.map == clone.map + assert datalayer.geojson != clone.geojson + assert datalayer.geojson.name != clone.geojson.name + assert clone.geojson.name == f"{clone.pk}.geojson" + + +def test_update_should_add_version(map, datalayer): + assert len(datalayer.versions) == 1 + datalayer.geojson = ContentFile("{}", "foo.json") + datalayer.save() + assert len(datalayer.versions) == 2 + + +def test_get_version(map, datalayer): + assert len(datalayer.versions) == 1 + datalayer.geojson = ContentFile('{"foo": "bar"}', "foo.json") + datalayer.save() + assert len(datalayer.versions) == 2 + latest = datalayer.versions[0]["ref"] + version = datalayer.get_version(latest) + assert json.loads(version) == {"foo": "bar"} + older = datalayer.versions[1]["ref"] + version = datalayer.get_version(older) + assert json.loads(version) == { + "_umap_options": { + "browsable": True, + "displayOnLoad": True, + "name": "test datalayer", + }, + "features": [ + { + "geometry": { + "coordinates": [ + 14.68896484375, + 48.55297816440071, + ], + "type": "Point", + }, + "properties": { + "_umap_options": { + "color": "DarkCyan", + "iconClass": "Ball", + }, + "description": "Da place anonymous again 755", + "name": "Here", + }, + "type": "Feature", + }, + ], + "type": "FeatureCollection", + } + + latest = datalayer.reference_version + version = datalayer.get_version(latest) + assert json.loads(version) == {"foo": "bar"} + + +def test_delete_datalayer_should_delete_all_versions(datalayer): + # create a new version + datalayer.geojson = ContentFile('{"foo": "bar"}', "foo.json") + datalayer.save() + s3_key = datalayer.geojson.name + datalayer.delete() + with pytest.raises(ClientError): + datalayer.geojson.storage.connection.meta.client.get_object( + Bucket="umap", + Key=s3_key, + )