From c0e91578b1f180de174fea9a6c95ff9ef21f556c Mon Sep 17 00:00:00 2001 From: Remco Schoen Date: Fri, 15 May 2020 17:59:08 +0000 Subject: [PATCH 01/26] Added docker support --- Dockerfile | 73 +++++++++++++++++++ docker-entrypoint.sh | 31 ++++++++ requirements-docker.txt | 4 ++ umap/settings/docker.py | 155 ++++++++++++++++++++++++++++++++++++++++ uwsgi.ini | 10 +++ 5 files changed, 273 insertions(+) create mode 100644 Dockerfile create mode 100755 docker-entrypoint.sh create mode 100644 requirements-docker.txt create mode 100644 umap/settings/docker.py create mode 100644 uwsgi.ini diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..51c6ea37 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,73 @@ +FROM node:12 AS vendors + +COPY . /srv/umap + +WORKDIR /srv/umap + +RUN make installjs + +RUN make vendors + +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED=1 \ + UMAP_SETTINGS=/srv/umap/umap/settings/docker.py \ + PORT=8000 + +RUN mkdir -p /srv/umap/data && \ + mkdir -p /srv/umap/uploads + +RUN mkdir /usr/share/man/man1 + +COPY . /srv/umap + +COPY --from=vendors /srv/umap/umap/static/umap/vendors /srv/umap/umap/static/umap/vendors + +WORKDIR /srv/umap + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + uwsgi \ + libpq-dev \ + build-essential \ + binutils \ + gdal-bin \ + libproj-dev \ + curl \ + git \ + gettext \ + sqlite3 \ + libffi-dev \ + libtiff5-dev \ + libjpeg62-turbo-dev \ + zlib1g-dev \ + libfreetype6-dev \ + liblcms2-dev \ + libwebp-dev \ + && \ + pip install --no-cache -r requirements-docker.txt && pip install . && \ + apt-get remove -y \ + binutils \ + libproj-dev \ + libffi-dev \ + libtiff5-dev \ + libjpeg62-turbo-dev \ + zlib1g-dev \ + libfreetype6-dev \ + liblcms2-dev \ + libwebp-dev \ + && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Add Tini +ENV TINI_VERSION v0.14.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini + +EXPOSE 8000 + +ENTRYPOINT ["/tini", "--"] + +CMD ["/srv/umap/docker-entrypoint.sh"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..c76416e0 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -eo pipefail + +# default variables +: "${SLEEP:=1}" +: "${TRIES:=60}" + +function wait_for_database {( + echo "Waiting for database to respond..." + tries=0 + while true; do + [[ $tries -lt $TRIES ]] || return + (echo "from django.db import connection; connection.connect()" | umap shell) >/dev/null 2>&1 + [[ $? -eq 0 ]] && return + sleep $SLEEP + tries=$((tries + 1)) + done +)} + +# first wait for the database +wait_for_database +# then migrate the database +umap migrate +# then collect static files +umap collectstatic --noinput +# create languagae files +#umap storagei18n +# compress static files +umap compress +# run uWSGI +exec uwsgi --ini uwsgi.ini diff --git a/requirements-docker.txt b/requirements-docker.txt new file mode 100644 index 00000000..92bb9611 --- /dev/null +++ b/requirements-docker.txt @@ -0,0 +1,4 @@ +-r requirements.txt +django-environ==0.4.1 +django-redis==4.7.0 +uwsgi==2.0.14 diff --git a/umap/settings/docker.py b/umap/settings/docker.py new file mode 100644 index 00000000..097a35da --- /dev/null +++ b/umap/settings/docker.py @@ -0,0 +1,155 @@ +# -*- coding:utf-8 -*- +""" +Settings for Docker development + +Use this file as a base for your local development settings and copy +it to umap/settings/local.py. It should not be checked into +your code repository. +""" +import environ +from umap.settings.base import * # pylint: disable=W0614,W0401 + +env = environ.Env() + +SECRET_KEY = env('SECRET_KEY') +INTERNAL_IPS = env.list('INTERNAL_IPS', default='127.0.0.1') +ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default='*') + +DEBUG = env.bool('DEBUG', default=False) + +ADMIN_EMAILS = env.list('ADMIN_EMAIL', default='') +ADMINS = [(email, email) for email in ADMIN_EMAILS] +MANAGERS = ADMINS + +DATABASES = { + 'default': env.db(default='postgis://localhost:5432/umap') +} + +COMPRESS_ENABLED = True +COMPRESS_OFFLINE = True + +LANGUAGE_CODE = 'en' + +# Set to False if login into django account should not be possible. You can +# administer accounts in the admin interface. +ENABLE_ACCOUNT_LOGIN = env.bool('ENABLE_ACCOUNT_LOGIN', default=True) + +AUTHENTICATION_BACKENDS = () + +# We need email to associate with other Oauth providers +SOCIAL_AUTH_GITHUB_SCOPE = ['user:email'] +SOCIAL_AUTH_GITHUB_KEY = env('GITHUB_KEY', default='') +SOCIAL_AUTH_GITHUB_SECRET = env('GITHUB_SECRET', default='') +if SOCIAL_AUTH_GITHUB_KEY and SOCIAL_AUTH_GITHUB_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.github.GithubOAuth2', + ) +SOCIAL_AUTH_BITBUCKET_KEY = env('BITBUCKET_KEY', default='') +SOCIAL_AUTH_BITBUCKET_SECRET = env('BITBUCKET_SECRET', default='') +if SOCIAL_AUTH_BITBUCKET_KEY and SOCIAL_AUTH_BITBUCKET_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.bitbucket.BitbucketOAuth', + ) + +SOCIAL_AUTH_TWITTER_KEY = env('TWITTER_KEY', default='') +SOCIAL_AUTH_TWITTER_SECRET = env('TWITTER_SECRET', default='') +if SOCIAL_AUTH_TWITTER_KEY and SOCIAL_AUTH_TWITTER_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.twitter.TwitterOAuth', + ) +SOCIAL_AUTH_OPENSTREETMAP_KEY = env('OPENSTREETMAP_KEY', default='') +SOCIAL_AUTH_OPENSTREETMAP_SECRET = env('OPENSTREETMAP_SECRET', default='') +if SOCIAL_AUTH_OPENSTREETMAP_KEY and SOCIAL_AUTH_OPENSTREETMAP_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.openstreetmap.OpenStreetMapOAuth', + ) + +AUTHENTICATION_BACKENDS += ( + 'django.contrib.auth.backends.ModelBackend', +) + +# MIDDLEWARE_CLASSES += ( +# 'social_django.middleware.SocialAuthExceptionMiddleware', +# ) + +SOCIAL_AUTH_RAISE_EXCEPTIONS = False +SOCIAL_AUTH_BACKEND_ERROR_URL = "/" + +# If you want to add a playgroud map, add its primary key +# UMAP_DEMO_PK = 204 +# If you want to add a showcase map on the home page, add its primary key +# UMAP_SHOWCASE_PK = 1156 +# Add a baner to warn people this instance is not production ready. +UMAP_DEMO_SITE = False + +# Whether to allow non authenticated people to create maps. +LEAFLET_STORAGE_ALLOW_ANONYMOUS = env.bool( + 'LEAFLET_STORAGE_ALLOW_ANONYMOUS', + default=False, +) + +# This setting will exclude empty maps (in fact, it will exclude all maps where +# the default center has not been updated) +UMAP_EXCLUDE_DEFAULT_MAPS = False + +# How many maps should be showcased on the main page resp. on the user page +UMAP_MAPS_PER_PAGE = 0 +# How many maps should be showcased on the user page, if owner +UMAP_MAPS_PER_PAGE_OWNER = 10 + +SITE_URL = env('SITE_URL') +SHORT_SITE_URL = env('SHORT_SITE_URL', default=None) + +CACHES = {'default': env.cache('REDIS_URL', default='locmem://')} + +# POSTGIS_VERSION = (2, 1, 0) +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + +# You need to unable accent extension before using UMAP_USE_UNACCENT +# python manage.py dbshell +# CREATE EXTENSION unaccent; +UMAP_USE_UNACCENT = False + +# For static deployment +STATIC_ROOT = '/srv/umap/static' + +# For users' statics (geojson mainly) +MEDIA_ROOT = '/srv/umap/uploads' + +# Default map location for new maps +LEAFLET_LONGITUDE = env.int('LEAFLET_LONGITUDE', default=2) +LEAFLET_LATITUDE = env.int('LEAFLET_LATITUDE', default=51) +LEAFLET_ZOOM = env.int('LEAFLET_ZOOM', default=6) + +# Number of old version to keep per datalayer. +LEAFLET_STORAGE_KEEP_VERSIONS = env.int( + 'LEAFLET_STORAGE_KEEP_VERSIONS', + default=10, +) + +import sys + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': '[django] %(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'verbose' + }, + }, + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': True, + }, + }, +} diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 00000000..2cf2c279 --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,10 @@ +[uwsgi] +http = :$(PORT) +module = umap.wsgi:application +master = True +vacuum = True +max-requests = 5000 +processes = 4 +enable-threads = true +static-map = /static=/srv/umap/static +static-map = /uploads=/srv/umap/uploads From 022ac18102497c6aebdf7e7d1e8ead730069e169 Mon Sep 17 00:00:00 2001 From: Duvel Date: Sat, 15 Oct 2022 15:09:09 +0200 Subject: [PATCH 02/26] Remove erroring line for man --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51c6ea37..9fbf8913 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,6 @@ ENV PYTHONUNBUFFERED=1 \ RUN mkdir -p /srv/umap/data && \ mkdir -p /srv/umap/uploads -RUN mkdir /usr/share/man/man1 - COPY . /srv/umap COPY --from=vendors /srv/umap/umap/static/umap/vendors /srv/umap/umap/static/umap/vendors From f2df2dd9c260ad0b18c87a03d3264a60cc77d87e Mon Sep 17 00:00:00 2001 From: Duvel Date: Sat, 15 Oct 2022 16:17:34 +0200 Subject: [PATCH 03/26] Requirements.txt has been removed --- requirements-docker.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-docker.txt b/requirements-docker.txt index 92bb9611..b1955e87 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,4 +1,3 @@ --r requirements.txt django-environ==0.4.1 django-redis==4.7.0 uwsgi==2.0.14 From 0cfd7c647b8e3085cdd3e73203c1c4ef7432242d Mon Sep 17 00:00:00 2001 From: Duvel Date: Sat, 15 Oct 2022 17:00:02 +0200 Subject: [PATCH 04/26] Prevent not displaying error --- docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index c76416e0..cf4094f0 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -10,7 +10,7 @@ function wait_for_database {( tries=0 while true; do [[ $tries -lt $TRIES ]] || return - (echo "from django.db import connection; connection.connect()" | umap shell) >/dev/null 2>&1 + (echo "from django.db import connection; connection.connect()" | umap shell) >/dev/null [[ $? -eq 0 ]] && return sleep $SLEEP tries=$((tries + 1)) From 2deb5597fdce7f6e2af86518c771f452f6afb7ec Mon Sep 17 00:00:00 2001 From: Duvel Date: Sun, 27 Nov 2022 16:21:26 +0100 Subject: [PATCH 05/26] Use latest Node JS LTS --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9fbf8913..3d2fad80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12 AS vendors +FROM node:18 AS vendors COPY . /srv/umap From 4f98297c99dc363302cc34b7683a2727c3d7625a Mon Sep 17 00:00:00 2001 From: Duvel Date: Sun, 27 Nov 2022 16:23:47 +0100 Subject: [PATCH 06/26] Removed suspended phantomjs and allows for arm64 --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 6503a1a3..19b1fc45 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,6 @@ "mocha": "^2.3.3", "mocha-phantomjs": "^4.0.1", "optimist": "~0.4.0", - "phantomjs": "^1.9.18", "sinon": "^1.10.3", "uglify-js": "~2.2.3" }, From c6677ce0c808cfa9de6df3bbd979cf12132ade05 Mon Sep 17 00:00:00 2001 From: Duvel Date: Sun, 27 Nov 2022 16:25:04 +0100 Subject: [PATCH 07/26] Copy statement didn't work in build container --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e0fc00c5..e6182deb 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,8 @@ vendors: mkdir -p umap/static/umap/vendors/georsstogeojson/ && cp -r node_modules/georsstogeojson/GeoRSSToGeoJSON.js umap/static/umap/vendors/georsstogeojson/ mkdir -p umap/static/umap/vendors/togpx/ && cp -r node_modules/togpx/togpx.js umap/static/umap/vendors/togpx/ mkdir -p umap/static/umap/vendors/tokml && cp -r node_modules/tokml/tokml.js umap/static/umap/vendors/tokml - mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/{dist/L.Control.Locate.css,src/L.Control.Locate.js} umap/static/umap/vendors/locatecontrol/ + mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/dist/L.Control.Locate.css umap/static/umap/vendors/locatecontrol/ + mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/src/L.Control.Locate.js umap/static/umap/vendors/locatecontrol/ installjs: npm install testjsfx: From 96e2c94e2db85de7e47ac41c8177965ac7c389d6 Mon Sep 17 00:00:00 2001 From: Duvel Date: Sun, 27 Nov 2022 17:01:08 +0100 Subject: [PATCH 08/26] Install tini with package manager to allow arm64 --- Dockerfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d2fad80..e8a523cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ WORKDIR /srv/umap RUN apt-get update && \ apt-get install -y --no-install-recommends \ + tini \ uwsgi \ libpq-dev \ build-essential \ @@ -59,13 +60,8 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# Add Tini -ENV TINI_VERSION v0.14.0 -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini -RUN chmod +x /tini - EXPOSE 8000 -ENTRYPOINT ["/tini", "--"] +ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/srv/umap/docker-entrypoint.sh"] From 5e201b44fbae2c6c3587c5f86dc2e3bfe186e763 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 11:48:21 +0200 Subject: [PATCH 09/26] Use django-environ to make part of the settings available throught env vars --- Dockerfile | 1 - setup.cfg | 1 + umap/settings/__init__.py | 53 +++++++------ umap/settings/base.py | 60 ++++++++++++--- umap/settings/docker.py | 155 -------------------------------------- 5 files changed, 75 insertions(+), 195 deletions(-) delete mode 100644 umap/settings/docker.py diff --git a/Dockerfile b/Dockerfile index e8a523cf..02926cbf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ RUN make vendors FROM python:3.8-slim ENV PYTHONUNBUFFERED=1 \ - UMAP_SETTINGS=/srv/umap/umap/settings/docker.py \ PORT=8000 RUN mkdir -p /srv/umap/data && \ diff --git a/setup.cfg b/setup.cfg index caa1e364..0db56af5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ install_requires = Django>=4.1 django-agnocomplete==2.2.0 django-compressor==4.3.1 + django-environ==0.10.0 Pillow==9.5.0 psycopg2==2.9.6 requests==2.30.0 diff --git a/umap/settings/__init__.py b/umap/settings/__init__.py index fc25bed1..aef9ce63 100644 --- a/umap/settings/__init__.py +++ b/umap/settings/__init__.py @@ -17,31 +17,30 @@ if not path: path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'local.py') if not os.path.exists(path): - msg = ('You must configure UMAP_SETTINGS or define ' - '/etc/umap/umap.conf') - print(colorize(msg, fg='red')) - sys.exit(1) + print(colorize('No valid UMAP_SETTINGS found', fg='yellow')) + path = None -d = types.ModuleType('config') -d.__file__ = path -try: - with open(path) as config_file: - exec(compile(config_file.read(), path, 'exec'), d.__dict__) -except IOError as e: - msg = 'Unable to import {} from UMAP_SETTINGS'.format(path) - print(colorize(msg, fg='red')) - sys.exit(e) -else: - print('Loaded local config from', path) - for key in dir(d): - if key.isupper(): - value = getattr(d, key) - if key.startswith('LEAFLET_STORAGE'): - # Retrocompat pre 1.0, remove me in 1.1. - globals()['UMAP' + key[15:]] = value - elif key == 'UMAP_CUSTOM_TEMPLATES': - globals()['TEMPLATES'][0]['DIRS'].insert(0, value) - elif key == 'UMAP_CUSTOM_STATICS': - globals()['STATICFILES_DIRS'].insert(0, value) - else: - globals()[key] = value +if path: + d = types.ModuleType('config') + d.__file__ = path + try: + with open(path) as config_file: + exec(compile(config_file.read(), path, 'exec'), d.__dict__) + except IOError as e: + msg = 'Unable to import {} from UMAP_SETTINGS'.format(path) + print(colorize(msg, fg='red')) + sys.exit(e) + else: + print('Loaded local config from', path) + for key in dir(d): + if key.isupper(): + value = getattr(d, key) + if key.startswith('LEAFLET_STORAGE'): + # Retrocompat pre 1.0, remove me in 1.1. + globals()['UMAP' + key[15:]] = value + elif key == 'UMAP_CUSTOM_TEMPLATES': + globals()['TEMPLATES'][0]['DIRS'].insert(0, value) + elif key == 'UMAP_CUSTOM_STATICS': + globals()['STATICFILES_DIRS'].insert(0, value) + else: + globals()[key] = value diff --git a/umap/settings/base.py b/umap/settings/base.py index 10b91139..07caa29a 100644 --- a/umap/settings/base.py +++ b/umap/settings/base.py @@ -1,13 +1,24 @@ """Base settings shared by all environments""" # Import global settings to make it easier to extend settings. +from email.utils import parseaddr + from django.template.defaultfilters import slugify from django.conf.locale import LANG_INFO +import environ + +env = environ.Env() # ============================================================================= # Generic Django project settings # ============================================================================= -DEBUG = True + +INTERNAL_IPS = env.list('INTERNAL_IPS', default='127.0.0.1') +ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*']) +ADMINS = tuple(parseaddr(email) for email in env.list('ADMINS', default=[])) + + +DEBUG = env.bool('DEBUG', default=False) SITE_ID = 1 # Add languages we're missing from Django @@ -92,7 +103,7 @@ LANGUAGES = ( ) # Make this unique, and don't share it with anybody. -SECRET_KEY = '' +SECRET_KEY = env('SECRET_KEY', default=None) INSTALLED_APPS = ( 'django.contrib.auth', @@ -197,21 +208,25 @@ MIDDLEWARE = ( # Auth / security # ============================================================================= -ENABLE_ACCOUNT_LOGIN = False +# Set to True if login into django account should be possible. Default is to +# only use OAuth flow. +ENABLE_ACCOUNT_LOGIN = env.bool("ENABLE_ACCOUNT_LOGIN", default=False) # ============================================================================= # Miscellaneous project settings # ============================================================================= -UMAP_ALLOW_ANONYMOUS = False +UMAP_ALLOW_ANONYMOUS = env.bool("UMAP_ALLOW_ANONYMOUS", default=False) + UMAP_EXTRA_URLS = { 'routing': 'http://www.openstreetmap.org/directions?engine=osrm_car&route={lat},{lng}&locale={locale}#map={zoom}/{lat}/{lng}', # noqa 'ajax_proxy': '/ajax-proxy/?url={url}&ttl={ttl}', 'search': 'https://photon.komoot.io/api/?', } -UMAP_KEEP_VERSIONS = 10 -SITE_URL = "http://umap.org" +UMAP_KEEP_VERSIONS = env.int('UMAP_KEEP_VERSIONS', default=10) +SITE_URL = env("SITE_URL", default="http://umap.org") +SHORT_SITE_URL = env('SHORT_SITE_URL', default=None) SITE_NAME = 'uMap' -UMAP_DEMO_SITE = False +UMAP_DEMO_SITE = env('UMAP_DEMO_SITE', default=False) UMAP_EXCLUDE_DEFAULT_MAPS = False UMAP_MAPS_PER_PAGE = 5 UMAP_MAPS_PER_PAGE_OWNER = 10 @@ -219,15 +234,18 @@ UMAP_SEARCH_CONFIGURATION = "simple" UMAP_FEEDBACK_LINK = "https://wiki.openstreetmap.org/wiki/UMap#Feedback_and_help" # noqa USER_MAPS_URL = 'user_maps' DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': 'umap', - } + 'default': env.db(default='postgis://localhost:5432/umap') } -UMAP_READONLY = False + +UMAP_READONLY = env('UMAP_READONLY', default=False) UMAP_GZIP = True LOCALE_PATHS = [os.path.join(PROJECT_DIR, 'locale')] +LEAFLET_LONGITUDE = env.int('LEAFLET_LONGITUDE', default=2) +LEAFLET_LATITUDE = env.int('LEAFLET_LATITUDE', default=51) +LEAFLET_ZOOM = env.int('LEAFLET_ZOOM', default=6) + + # ============================================================================= # Third party app settings # ============================================================================= @@ -240,3 +258,21 @@ SOCIAL_AUTH_NO_DEFAULT_PROTECTED_USER_FIELDS = True SOCIAL_AUTH_PROTECTED_USER_FIELDS = ("id", ) LOGIN_URL = "login" SOCIAL_AUTH_LOGIN_REDIRECT_URL = "/login/popup/end/" + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'level': 'ERROR', + 'filters': None, + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': 'ERROR', + }, + }, +} diff --git a/umap/settings/docker.py b/umap/settings/docker.py deleted file mode 100644 index 097a35da..00000000 --- a/umap/settings/docker.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding:utf-8 -*- -""" -Settings for Docker development - -Use this file as a base for your local development settings and copy -it to umap/settings/local.py. It should not be checked into -your code repository. -""" -import environ -from umap.settings.base import * # pylint: disable=W0614,W0401 - -env = environ.Env() - -SECRET_KEY = env('SECRET_KEY') -INTERNAL_IPS = env.list('INTERNAL_IPS', default='127.0.0.1') -ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default='*') - -DEBUG = env.bool('DEBUG', default=False) - -ADMIN_EMAILS = env.list('ADMIN_EMAIL', default='') -ADMINS = [(email, email) for email in ADMIN_EMAILS] -MANAGERS = ADMINS - -DATABASES = { - 'default': env.db(default='postgis://localhost:5432/umap') -} - -COMPRESS_ENABLED = True -COMPRESS_OFFLINE = True - -LANGUAGE_CODE = 'en' - -# Set to False if login into django account should not be possible. You can -# administer accounts in the admin interface. -ENABLE_ACCOUNT_LOGIN = env.bool('ENABLE_ACCOUNT_LOGIN', default=True) - -AUTHENTICATION_BACKENDS = () - -# We need email to associate with other Oauth providers -SOCIAL_AUTH_GITHUB_SCOPE = ['user:email'] -SOCIAL_AUTH_GITHUB_KEY = env('GITHUB_KEY', default='') -SOCIAL_AUTH_GITHUB_SECRET = env('GITHUB_SECRET', default='') -if SOCIAL_AUTH_GITHUB_KEY and SOCIAL_AUTH_GITHUB_SECRET: - AUTHENTICATION_BACKENDS += ( - 'social_core.backends.github.GithubOAuth2', - ) -SOCIAL_AUTH_BITBUCKET_KEY = env('BITBUCKET_KEY', default='') -SOCIAL_AUTH_BITBUCKET_SECRET = env('BITBUCKET_SECRET', default='') -if SOCIAL_AUTH_BITBUCKET_KEY and SOCIAL_AUTH_BITBUCKET_SECRET: - AUTHENTICATION_BACKENDS += ( - 'social_core.backends.bitbucket.BitbucketOAuth', - ) - -SOCIAL_AUTH_TWITTER_KEY = env('TWITTER_KEY', default='') -SOCIAL_AUTH_TWITTER_SECRET = env('TWITTER_SECRET', default='') -if SOCIAL_AUTH_TWITTER_KEY and SOCIAL_AUTH_TWITTER_SECRET: - AUTHENTICATION_BACKENDS += ( - 'social_core.backends.twitter.TwitterOAuth', - ) -SOCIAL_AUTH_OPENSTREETMAP_KEY = env('OPENSTREETMAP_KEY', default='') -SOCIAL_AUTH_OPENSTREETMAP_SECRET = env('OPENSTREETMAP_SECRET', default='') -if SOCIAL_AUTH_OPENSTREETMAP_KEY and SOCIAL_AUTH_OPENSTREETMAP_SECRET: - AUTHENTICATION_BACKENDS += ( - 'social_core.backends.openstreetmap.OpenStreetMapOAuth', - ) - -AUTHENTICATION_BACKENDS += ( - 'django.contrib.auth.backends.ModelBackend', -) - -# MIDDLEWARE_CLASSES += ( -# 'social_django.middleware.SocialAuthExceptionMiddleware', -# ) - -SOCIAL_AUTH_RAISE_EXCEPTIONS = False -SOCIAL_AUTH_BACKEND_ERROR_URL = "/" - -# If you want to add a playgroud map, add its primary key -# UMAP_DEMO_PK = 204 -# If you want to add a showcase map on the home page, add its primary key -# UMAP_SHOWCASE_PK = 1156 -# Add a baner to warn people this instance is not production ready. -UMAP_DEMO_SITE = False - -# Whether to allow non authenticated people to create maps. -LEAFLET_STORAGE_ALLOW_ANONYMOUS = env.bool( - 'LEAFLET_STORAGE_ALLOW_ANONYMOUS', - default=False, -) - -# This setting will exclude empty maps (in fact, it will exclude all maps where -# the default center has not been updated) -UMAP_EXCLUDE_DEFAULT_MAPS = False - -# How many maps should be showcased on the main page resp. on the user page -UMAP_MAPS_PER_PAGE = 0 -# How many maps should be showcased on the user page, if owner -UMAP_MAPS_PER_PAGE_OWNER = 10 - -SITE_URL = env('SITE_URL') -SHORT_SITE_URL = env('SHORT_SITE_URL', default=None) - -CACHES = {'default': env.cache('REDIS_URL', default='locmem://')} - -# POSTGIS_VERSION = (2, 1, 0) -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' - -# You need to unable accent extension before using UMAP_USE_UNACCENT -# python manage.py dbshell -# CREATE EXTENSION unaccent; -UMAP_USE_UNACCENT = False - -# For static deployment -STATIC_ROOT = '/srv/umap/static' - -# For users' statics (geojson mainly) -MEDIA_ROOT = '/srv/umap/uploads' - -# Default map location for new maps -LEAFLET_LONGITUDE = env.int('LEAFLET_LONGITUDE', default=2) -LEAFLET_LATITUDE = env.int('LEAFLET_LATITUDE', default=51) -LEAFLET_ZOOM = env.int('LEAFLET_ZOOM', default=6) - -# Number of old version to keep per datalayer. -LEAFLET_STORAGE_KEEP_VERSIONS = env.int( - 'LEAFLET_STORAGE_KEEP_VERSIONS', - default=10, -) - -import sys - -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'verbose': { - 'format': '[django] %(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' - } - }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'stream': sys.stdout, - 'formatter': 'verbose' - }, - }, - 'loggers': { - 'django': { - 'handlers': ['console'], - 'level': 'DEBUG', - 'propagate': True, - }, - }, -} From 40f694f460ce982c5dd9b7552def3f9a880af167 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 11:48:46 +0200 Subject: [PATCH 10/26] Make Docker happy with Makefile syntax --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cadc786f..4968d495 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,8 @@ vendors: mkdir -p umap/static/umap/vendors/editable/ && cp -r node_modules/leaflet.path.drag/src/*.js umap/static/umap/vendors/editable/ mkdir -p umap/static/umap/vendors/hash/ && cp -r node_modules/leaflet-hash/*.js umap/static/umap/vendors/hash/ mkdir -p umap/static/umap/vendors/i18n/ && cp -r node_modules/leaflet-i18n/*.js umap/static/umap/vendors/i18n/ - mkdir -p umap/static/umap/vendors/editinosm/ && cp -r node_modules/leaflet-editinosm/{Leaflet.EditInOSM.*,edit-in-osm.png} umap/static/umap/vendors/editinosm/ + mkdir -p umap/static/umap/vendors/editinosm/ && cp -r node_modules/leaflet-editinosm/Leaflet.EditInOSM.* umap/static/umap/vendors/editinosm/ + mkdir -p umap/static/umap/vendors/editinosm/ && cp -r node_modules/leaflet-editinosm/edit-in-osm.png umap/static/umap/vendors/editinosm/ mkdir -p umap/static/umap/vendors/minimap/ && cp -r node_modules/leaflet-minimap/src/** umap/static/umap/vendors/minimap/ mkdir -p umap/static/umap/vendors/loading/ && cp -r node_modules/leaflet-loading/src/** umap/static/umap/vendors/loading/ mkdir -p umap/static/umap/vendors/markercluster/ && cp -r node_modules/leaflet.markercluster/dist/** umap/static/umap/vendors/markercluster/ From 96ca298a288a915179c16cf7dcf7ba6e661c7ff8 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 11:49:21 +0200 Subject: [PATCH 11/26] Remove unused code --- docker-entrypoint.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index cf4094f0..3619636d 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -23,8 +23,6 @@ wait_for_database umap migrate # then collect static files umap collectstatic --noinput -# create languagae files -#umap storagei18n # compress static files umap compress # run uWSGI From cf071eda859233709b9a6c6ddff36237f2fd0bbc Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 11:49:36 +0200 Subject: [PATCH 12/26] Remove mocha-phantomjs for now, as it sometimes break Docker build --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 97480e95..09acd09d 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,6 @@ "happen": "~0.1.3", "lebab": "^3.2.1", "mocha": "^2.3.3", - "mocha-phantomjs": "^4.0.1", "optimist": "~0.4.0", "prettier": "^2.8.8", "sinon": "^15.1.0", From 24d4a5d488a0bb429993539a44d56f141b44568a Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 12:03:42 +0200 Subject: [PATCH 13/26] Remove requirements-docker.txt --- Dockerfile | 2 +- requirements-docker.txt | 3 --- setup.cfg | 3 ++- 3 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 requirements-docker.txt diff --git a/Dockerfile b/Dockerfile index 02926cbf..15a3ed31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && \ liblcms2-dev \ libwebp-dev \ && \ - pip install --no-cache -r requirements-docker.txt && pip install . && \ + pip install .[docker] && \ apt-get remove -y \ binutils \ libproj-dev \ diff --git a/requirements-docker.txt b/requirements-docker.txt deleted file mode 100644 index b1955e87..00000000 --- a/requirements-docker.txt +++ /dev/null @@ -1,3 +0,0 @@ -django-environ==0.4.1 -django-redis==4.7.0 -uwsgi==2.0.14 diff --git a/setup.cfg b/setup.cfg index 0db56af5..01292336 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,8 @@ test = factory-boy==3.2.1 pytest==6.2.5 pytest-django==4.5.2 - +docker = + uwsgi==2.0.21 [options.entry_points] console_scripts = From e322358b51c00930edfc7f6f44d14fe502baaa0c Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 12:03:54 +0200 Subject: [PATCH 14/26] Add .dockerignore --- .dockerignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..78de9126 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +tmp/ +build/ +dist/ +*.egg-info/ +node_modules/ +umap/settings/local.py From b749b734c7a0ab5527d3efcba7ac25b862ab5f14 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 12:17:45 +0200 Subject: [PATCH 15/26] Use python 3.11 in docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 15a3ed31..0e00c1cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN make installjs RUN make vendors -FROM python:3.8-slim +FROM python:3.11-slim ENV PYTHONUNBUFFERED=1 \ PORT=8000 From 76ed5b9c4f457d3dfb369ab0f842bcfb1efee527 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 12:26:50 +0200 Subject: [PATCH 16/26] Move docker utils under a dedicated folder --- Dockerfile | 2 +- docker-entrypoint.sh => docker/entrypoint.sh | 2 +- uwsgi.ini => docker/uwsgi.ini | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename docker-entrypoint.sh => docker/entrypoint.sh (94%) rename uwsgi.ini => docker/uwsgi.ini (100%) diff --git a/Dockerfile b/Dockerfile index 0e00c1cd..3fedeab8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,4 +63,4 @@ EXPOSE 8000 ENTRYPOINT ["/usr/bin/tini", "--"] -CMD ["/srv/umap/docker-entrypoint.sh"] +CMD ["/srv/umap/docker/entrypoint.sh"] diff --git a/docker-entrypoint.sh b/docker/entrypoint.sh similarity index 94% rename from docker-entrypoint.sh rename to docker/entrypoint.sh index 3619636d..704509f4 100755 --- a/docker-entrypoint.sh +++ b/docker/entrypoint.sh @@ -26,4 +26,4 @@ umap collectstatic --noinput # compress static files umap compress # run uWSGI -exec uwsgi --ini uwsgi.ini +exec uwsgi --ini docker/uwsgi.ini diff --git a/uwsgi.ini b/docker/uwsgi.ini similarity index 100% rename from uwsgi.ini rename to docker/uwsgi.ini From e1d3a95cceefff74ba12237d14c30c8df9379f40 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 13:19:14 +0200 Subject: [PATCH 17/26] Allow to make vendors without Makefile So we can use a lighter node image in Docker --- Dockerfile | 6 +++--- Makefile | 27 +-------------------------- package.json | 2 +- scripts/vendorsjs.sh | 30 ++++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 30 deletions(-) create mode 100755 scripts/vendorsjs.sh diff --git a/Dockerfile b/Dockerfile index 3fedeab8..19bc1dbb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM node:18 AS vendors +FROM node:alpine AS vendors COPY . /srv/umap WORKDIR /srv/umap -RUN make installjs +RUN npm install -RUN make vendors +RUN npm run vendors FROM python:3.11-slim diff --git a/Makefile b/Makefile index 4968d495..16166160 100644 --- a/Makefile +++ b/Makefile @@ -19,32 +19,7 @@ messages: cd umap && umap makemessages -l en node node_modules/leaflet-i18n/bin/i18n.js --dir_path=umap/static/umap/js/ --dir_path=umap/static/umap/vendors/measurable/ --locale_dir_path=umap/static/umap/locale/ --locale_codes=en --mode=json --clean --default_values vendors: - mkdir -p umap/static/umap/vendors/leaflet/ && cp -r node_modules/leaflet/dist/** umap/static/umap/vendors/leaflet/ - mkdir -p umap/static/umap/vendors/editable/ && cp -r node_modules/leaflet-editable/src/*.js umap/static/umap/vendors/editable/ - mkdir -p umap/static/umap/vendors/editable/ && cp -r node_modules/leaflet.path.drag/src/*.js umap/static/umap/vendors/editable/ - mkdir -p umap/static/umap/vendors/hash/ && cp -r node_modules/leaflet-hash/*.js umap/static/umap/vendors/hash/ - mkdir -p umap/static/umap/vendors/i18n/ && cp -r node_modules/leaflet-i18n/*.js umap/static/umap/vendors/i18n/ - mkdir -p umap/static/umap/vendors/editinosm/ && cp -r node_modules/leaflet-editinosm/Leaflet.EditInOSM.* umap/static/umap/vendors/editinosm/ - mkdir -p umap/static/umap/vendors/editinosm/ && cp -r node_modules/leaflet-editinosm/edit-in-osm.png umap/static/umap/vendors/editinosm/ - mkdir -p umap/static/umap/vendors/minimap/ && cp -r node_modules/leaflet-minimap/src/** umap/static/umap/vendors/minimap/ - mkdir -p umap/static/umap/vendors/loading/ && cp -r node_modules/leaflet-loading/src/** umap/static/umap/vendors/loading/ - mkdir -p umap/static/umap/vendors/markercluster/ && cp -r node_modules/leaflet.markercluster/dist/** umap/static/umap/vendors/markercluster/ - mkdir -p umap/static/umap/vendors/contextmenu/ && cp -r node_modules/leaflet-contextmenu/dist/** umap/static/umap/vendors/contextmenu/ - mkdir -p umap/static/umap/vendors/heat/ && cp -r node_modules/leaflet.heat/dist/** umap/static/umap/vendors/heat/ - mkdir -p umap/static/umap/vendors/fullscreen/ && cp -r node_modules/leaflet-fullscreen/dist/** umap/static/umap/vendors/fullscreen/ - mkdir -p umap/static/umap/vendors/toolbar/ && cp -r node_modules/leaflet-toolbar/dist/** umap/static/umap/vendors/toolbar/ - mkdir -p umap/static/umap/vendors/formbuilder/ && cp -r node_modules/leaflet-formbuilder/*.js umap/static/umap/vendors/formbuilder/ - mkdir -p umap/static/umap/vendors/measurable/ && cp -r node_modules/leaflet-measurable/Leaflet.Measurable.* umap/static/umap/vendors/measurable/ - mkdir -p umap/static/umap/vendors/photon/ && cp -r node_modules/leaflet.photon/*.js umap/static/umap/vendors/photon/ - mkdir -p umap/static/umap/vendors/csv2geojson/ && cp -r node_modules/csv2geojson/*.js umap/static/umap/vendors/csv2geojson/ - mkdir -p umap/static/umap/vendors/togeojson/ && cp -r node_modules/togeojson/*.js umap/static/umap/vendors/togeojson/ - mkdir -p umap/static/umap/vendors/osmtogeojson/ && cp -r node_modules/osmtogeojson/osmtogeojson.js umap/static/umap/vendors/osmtogeojson/ - mkdir -p umap/static/umap/vendors/georsstogeojson/ && cp -r node_modules/georsstogeojson/GeoRSSToGeoJSON.js umap/static/umap/vendors/georsstogeojson/ - mkdir -p umap/static/umap/vendors/togpx/ && cp -r node_modules/togpx/togpx.js umap/static/umap/vendors/togpx/ - mkdir -p umap/static/umap/vendors/tokml && cp -r node_modules/tokml/tokml.js umap/static/umap/vendors/tokml - mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/dist/L.Control.Locate.css umap/static/umap/vendors/locatecontrol/ - mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/src/L.Control.Locate.js umap/static/umap/vendors/locatecontrol/ - mkdir -p umap/static/umap/vendors/dompurify/ && cp -r node_modules/dompurify/dist/purify.js umap/static/umap/vendors/dompurify/ + npm run vendors installjs: npm install testjsfx: diff --git a/package.json b/package.json index 09acd09d..94c1ed9f 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ }, "scripts": { "test": "firefox test/index.html", - "build": "grunt" + "vendors": "scripts/vendorsjs.sh" }, "repository": { "type": "git", diff --git a/scripts/vendorsjs.sh b/scripts/vendorsjs.sh new file mode 100755 index 00000000..de375479 --- /dev/null +++ b/scripts/vendorsjs.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh + +mkdir -p umap/static/umap/vendors/leaflet/ && cp -r node_modules/leaflet/dist/** umap/static/umap/vendors/leaflet/ +mkdir -p umap/static/umap/vendors/editable/ && cp -r node_modules/leaflet-editable/src/*.js umap/static/umap/vendors/editable/ +mkdir -p umap/static/umap/vendors/editable/ && cp -r node_modules/leaflet.path.drag/src/*.js umap/static/umap/vendors/editable/ +mkdir -p umap/static/umap/vendors/hash/ && cp -r node_modules/leaflet-hash/*.js umap/static/umap/vendors/hash/ +mkdir -p umap/static/umap/vendors/i18n/ && cp -r node_modules/leaflet-i18n/*.js umap/static/umap/vendors/i18n/ +mkdir -p umap/static/umap/vendors/editinosm/ && cp -r node_modules/leaflet-editinosm/Leaflet.EditInOSM.* umap/static/umap/vendors/editinosm/ +mkdir -p umap/static/umap/vendors/editinosm/ && cp -r node_modules/leaflet-editinosm/edit-in-osm.png umap/static/umap/vendors/editinosm/ +mkdir -p umap/static/umap/vendors/minimap/ && cp -r node_modules/leaflet-minimap/src/** umap/static/umap/vendors/minimap/ +mkdir -p umap/static/umap/vendors/loading/ && cp -r node_modules/leaflet-loading/src/** umap/static/umap/vendors/loading/ +mkdir -p umap/static/umap/vendors/markercluster/ && cp -r node_modules/leaflet.markercluster/dist/** umap/static/umap/vendors/markercluster/ +mkdir -p umap/static/umap/vendors/contextmenu/ && cp -r node_modules/leaflet-contextmenu/dist/** umap/static/umap/vendors/contextmenu/ +mkdir -p umap/static/umap/vendors/heat/ && cp -r node_modules/leaflet.heat/dist/** umap/static/umap/vendors/heat/ +mkdir -p umap/static/umap/vendors/fullscreen/ && cp -r node_modules/leaflet-fullscreen/dist/** umap/static/umap/vendors/fullscreen/ +mkdir -p umap/static/umap/vendors/toolbar/ && cp -r node_modules/leaflet-toolbar/dist/** umap/static/umap/vendors/toolbar/ +mkdir -p umap/static/umap/vendors/formbuilder/ && cp -r node_modules/leaflet-formbuilder/*.js umap/static/umap/vendors/formbuilder/ +mkdir -p umap/static/umap/vendors/measurable/ && cp -r node_modules/leaflet-measurable/Leaflet.Measurable.* umap/static/umap/vendors/measurable/ +mkdir -p umap/static/umap/vendors/photon/ && cp -r node_modules/leaflet.photon/*.js umap/static/umap/vendors/photon/ +mkdir -p umap/static/umap/vendors/csv2geojson/ && cp -r node_modules/csv2geojson/*.js umap/static/umap/vendors/csv2geojson/ +mkdir -p umap/static/umap/vendors/togeojson/ && cp -r node_modules/togeojson/*.js umap/static/umap/vendors/togeojson/ +mkdir -p umap/static/umap/vendors/osmtogeojson/ && cp -r node_modules/osmtogeojson/osmtogeojson.js umap/static/umap/vendors/osmtogeojson/ +mkdir -p umap/static/umap/vendors/georsstogeojson/ && cp -r node_modules/georsstogeojson/GeoRSSToGeoJSON.js umap/static/umap/vendors/georsstogeojson/ +mkdir -p umap/static/umap/vendors/togpx/ && cp -r node_modules/togpx/togpx.js umap/static/umap/vendors/togpx/ +mkdir -p umap/static/umap/vendors/tokml && cp -r node_modules/tokml/tokml.js umap/static/umap/vendors/tokml +mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/dist/L.Control.Locate.css umap/static/umap/vendors/locatecontrol/ +mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/src/L.Control.Locate.js umap/static/umap/vendors/locatecontrol/ +mkdir -p umap/static/umap/vendors/dompurify/ && cp -r node_modules/dompurify/dist/purify.js umap/static/umap/vendors/dompurify/ + +echo 'Done!' From d7d47bf50819b6282b9ce36a7275bac452bcfb4c Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 13:57:32 +0200 Subject: [PATCH 18/26] Remove unused data folder from Dockerfile --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19bc1dbb..883663fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,7 @@ FROM python:3.11-slim ENV PYTHONUNBUFFERED=1 \ PORT=8000 -RUN mkdir -p /srv/umap/data && \ - mkdir -p /srv/umap/uploads +RUN mkdir -p /srv/umap/uploads COPY . /srv/umap From c665dba082212e2f9dcc3c3940dede412553586e Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 14:33:26 +0200 Subject: [PATCH 19/26] Add a basic docker-compose.yml as example --- docker-compose.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..6e8f4201 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3' +services: + db: + image: postgis/postgis:14-3.3-alpine + environment: + - POSTGRES_HOST_AUTH_METHOD=trust + volumes: + - db:/var/lib/postgresql/data + + app: + image: umap:1.3.0 + ports: + - "8001:8000" + environment: + - DATABASE_URL=postgis://postgres@db/postgres + - SECRET_KEY=some-long-and-weirdly-unrandom-secret-key + - SITE_URL=https://umap.local/ + - UMAP_ALLOW_ANONYMOUS=True + - DEBUG=1 + volumes: + - data:/srv/umap/uploads + +volumes: + data: + db: From 5190c46bf6bba6b7156186734cbec9c7d471bf7f Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 14:44:33 +0200 Subject: [PATCH 20/26] Do not track Dockerfile, not to invalidate Docker cache at each change --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 78de9126..c6df9055 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,5 @@ dist/ *.egg-info/ node_modules/ umap/settings/local.py +Dockerfile +.git/ From 00f34c45baa6a46a3757dcd0773a1a3eb1a32f8f Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 14:48:55 +0200 Subject: [PATCH 21/26] Handle OpenStreetMap OAuth by default --- umap/settings/base.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/umap/settings/base.py b/umap/settings/base.py index 07caa29a..0b86b55b 100644 --- a/umap/settings/base.py +++ b/umap/settings/base.py @@ -259,6 +259,19 @@ SOCIAL_AUTH_PROTECTED_USER_FIELDS = ("id", ) LOGIN_URL = "login" SOCIAL_AUTH_LOGIN_REDIRECT_URL = "/login/popup/end/" +AUTHENTICATION_BACKENDS = () + +SOCIAL_AUTH_OPENSTREETMAP_KEY = env('SOCIAL_AUTH_OPENSTREETMAP_KEY', default="") +SOCIAL_AUTH_OPENSTREETMAP_SECRET = env('SOCIAL_AUTH_OPENSTREETMAP_SECRET', default="") +if SOCIAL_AUTH_OPENSTREETMAP_KEY and SOCIAL_AUTH_OPENSTREETMAP_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.openstreetmap.OpenStreetMapOAuth', + ) + +AUTHENTICATION_BACKENDS += ( + 'django.contrib.auth.backends.ModelBackend', +) + LOGGING = { 'version': 1, 'disable_existing_loggers': False, From faebeebbdf948aae9c326522740d2617d39843e0 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 15:08:19 +0200 Subject: [PATCH 22/26] INTERNAL_IPS should default to a list --- umap/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/umap/settings/base.py b/umap/settings/base.py index 0b86b55b..9992b011 100644 --- a/umap/settings/base.py +++ b/umap/settings/base.py @@ -13,7 +13,7 @@ env = environ.Env() # ============================================================================= -INTERNAL_IPS = env.list('INTERNAL_IPS', default='127.0.0.1') +INTERNAL_IPS = env.list('INTERNAL_IPS', default=['127.0.0.1']) ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*']) ADMINS = tuple(parseaddr(email) for email in env.list('ADMINS', default=[])) From 3856cd447bc1d5bbeb5655fa3ab44748a97f5860 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 15:44:32 +0200 Subject: [PATCH 23/26] Use layers to build Docker image Better cache and lighter image, yay. --- Dockerfile | 82 +++++++++++++++++++++++--------------------- docker/entrypoint.sh | 2 ++ docker/uwsgi.ini | 1 + 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/Dockerfile b/Dockerfile index 883663fb..c8ccd9aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,56 +8,60 @@ RUN npm install RUN npm run vendors -FROM python:3.11-slim - -ENV PYTHONUNBUFFERED=1 \ - PORT=8000 - -RUN mkdir -p /srv/umap/uploads - -COPY . /srv/umap - -COPY --from=vendors /srv/umap/umap/static/umap/vendors /srv/umap/umap/static/umap/vendors - -WORKDIR /srv/umap +FROM python:3.11-slim as common RUN apt-get update && \ apt-get install -y --no-install-recommends \ tini \ uwsgi \ - libpq-dev \ - build-essential \ - binutils \ - gdal-bin \ - libproj-dev \ - curl \ - git \ - gettext \ sqlite3 \ - libffi-dev \ - libtiff5-dev \ - libjpeg62-turbo-dev \ - zlib1g-dev \ - libfreetype6-dev \ - liblcms2-dev \ - libwebp-dev \ - && \ - pip install .[docker] && \ - apt-get remove -y \ - binutils \ - libproj-dev \ - libffi-dev \ - libtiff5-dev \ - libjpeg62-turbo-dev \ - zlib1g-dev \ - libfreetype6-dev \ - liblcms2-dev \ - libwebp-dev \ + libpq-dev \ + gdal-bin \ && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +FROM common as build + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + binutils \ + libproj-dev \ + curl \ + git \ + gettext \ + python3-venv \ + libffi-dev \ + libtiff5-dev \ + libjpeg62-turbo-dev \ + zlib1g-dev \ + libfreetype6-dev \ + liblcms2-dev \ + libwebp-dev + +RUN python -m venv /venv + +WORKDIR /srv/umap + +COPY . /srv/umap + +RUN /venv/bin/pip install .[docker] + +FROM common + +COPY --from=build /srv/umap/docker/ /srv/umap/docker/ +COPY --from=build /venv/ /venv/ +COPY --from=vendors /srv/umap/umap/static/umap/vendors /srv/umap/umap/static/umap/vendors + +WORKDIR /srv/umap + +RUN mkdir -p /srv/umap/uploads + +ENV PYTHONUNBUFFERED=1 \ + PORT=8000 + EXPOSE 8000 ENTRYPOINT ["/usr/bin/tini", "--"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 704509f4..bf6c5165 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eo pipefail +source /venv/bin/activate + # default variables : "${SLEEP:=1}" : "${TRIES:=60}" diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 2cf2c279..2239f863 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -1,5 +1,6 @@ [uwsgi] http = :$(PORT) +home = /venv module = umap.wsgi:application master = True vacuum = True From efdda1adac5e753a30397262515b928df366aa95 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 16:09:12 +0200 Subject: [PATCH 24/26] Better cache npm install in Docker build I also add to install git, no idea why it was working before and not after this change :/ --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c8ccd9aa..63bb73fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,15 @@ FROM node:alpine AS vendors -COPY . /srv/umap +RUN apk add git WORKDIR /srv/umap +COPY package.json . + RUN npm install +COPY . . + RUN npm run vendors FROM python:3.11-slim as common From 8ce951dd539eab310e7eaa8ea1cb9222c306f13c Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 16:30:12 +0200 Subject: [PATCH 25/26] Add comments in Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 63bb73fc..883931d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ COPY . . RUN npm run vendors +# This part installs deps needed at runtime. FROM python:3.11-slim as common RUN apt-get update && \ @@ -26,6 +27,7 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# This part adds deps needed only at buildtime. FROM common as build RUN apt-get update && \ From 7dac5b2994b039f006b7e3e5d02138e40758cffc Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Mon, 5 Jun 2023 16:59:49 +0200 Subject: [PATCH 26/26] Very minimal doc for Docker install --- docs/docker.md | 33 +++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 34 insertions(+) create mode 100644 docs/docker.md diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 00000000..baceccb7 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,33 @@ +# Docker + +There is now an official [uMap](https://hub.docker.com/r/umap/umap) image. + +To run it with docker compose, use a `docker-compose.yml` like this: + +```yaml +version: '3' +services: + db: + image: postgis/postgis:14-3.3-alpine + environment: + - POSTGRES_HOST_AUTH_METHOD=trust + volumes: + - db:/var/lib/postgresql/data + + app: + image: umap/umap:x.x.x + ports: + - "8001:8000" + environment: + - DATABASE_URL=postgis://postgres@db/postgres + - SECRET_KEY=some-long-and-weirdly-unrandom-secret-key + - SITE_URL=https://umap.local/ + - UMAP_ALLOW_ANONYMOUS=True + - DEBUG=1 + volumes: + - data:/srv/umap/uploads + +volumes: + data: + db: +``` diff --git a/mkdocs.yml b/mkdocs.yml index 1293c4ae..c09cc09d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,5 +7,6 @@ nav: - how-tos: - Ubuntu from scratch: ubuntu.md - Customize your uMap style: custom.md + - Install with Docker: docker.md - Changelog: changelog.md theme: readthedocs