From 0cffe1e174eb7d5a70833d702f49726795502cfb Mon Sep 17 00:00:00 2001 From: Nikita Klyshko Date: Sat, 15 Apr 2023 16:04:21 +0300 Subject: [PATCH] Closes #1: Add docker environment for development and production (#12) * Added docker support * Remove erroring line for man * Requirements.txt has been removed * Prevent not displaying error * Use latest Node JS LTS * Removed suspended phantomjs and allows for arm64 * Copy statement didn't work in build container * Install tini with package manager to allow arm64 * Closes #1: Add docker environment for development and production * Fix --------- Co-authored-by: Remco Schoen --- .dockerignore | 6 ++ .env.example | 4 + .gitignore | 5 ++ .vscode/launch.json | 24 ++++++ Dockerfile | 70 ++++++++++++++++ Makefile | 3 +- docker-compose.override.yml | 14 ++++ docker-compose.yml | 28 +++++++ docker-entrypoint.sh | 35 ++++++++ package.json | 1 - requirements-docker.txt | 3 + umap/settings/dev.py | 1 + umap/settings/docker.py | 155 ++++++++++++++++++++++++++++++++++++ uwsgi.ini | 10 +++ 14 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .vscode/launch.json create mode 100644 Dockerfile create mode 100644 docker-compose.override.yml create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh create mode 100644 requirements-docker.txt create mode 100644 umap/settings/docker.py create mode 100644 uwsgi.ini diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..faac0e23 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.vscode/ +.venv/ +build/ +static/ +umap_project.egg-info/ +data/ diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..ec55279e --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +SECRET_KEY=some-long-and-weirdly-unrandom-secret-key +SITE_URL=https://umap.local/ +DATABASE_USER=postgres +DATABASE_PASSWORD=postgres diff --git a/.gitignore b/.gitignore index d236ae74..2dd0996e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ node_modules/* umap/static/umap/vendors site/* .pytest_cache/ +static/ +uploads/ ### Python ### # Byte-compiled / optimized / DLL files @@ -18,3 +20,6 @@ build/ dist/ *.egg-info/ +.env +.venv/ + diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..b4627221 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Remote Attach", + "type": "python", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/umap/", + "remoteRoot": "/srv/app/umap/" + } + ], + "justMyCode": false + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..34df0aee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,70 @@ +FROM node:18 AS vendors + +COPY . /srv/app + +WORKDIR /srv/app + +RUN make installjs +RUN make vendors + +FROM python:3.8-slim as app_python + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + tini \ + uwsgi \ + libpq-dev \ + build-essential \ + binutils \ + gdal-bin \ + libproj-dev \ + curl \ + git \ + gettext \ + sqlite3 \ + libffi-dev \ + libtiff5-dev \ + libjpeg62-turbo-dev \ + zlib1g-dev \ + libfreetype6-dev \ + liblcms2-dev \ + libwebp-dev + +ENV PYTHONUNBUFFERED=1 \ + UMAP_SETTINGS=/srv/app/umap/settings/docker.py \ + PORT=8000 + +COPY . /srv/app +RUN mkdir -p /srv/app/data && \ + mkdir -p /srv/app/uploads +COPY --from=vendors /srv/app/umap/static/umap/vendors /srv/app/umap/static/umap/vendors + +WORKDIR /srv/app + +RUN pip install --no-cache -r requirements-docker.txt && pip install . +RUN apt-get remove -y \ + binutils \ + libproj-dev \ + libffi-dev \ + libtiff5-dev \ + libjpeg62-turbo-dev \ + zlib1g-dev \ + libfreetype6-dev \ + liblcms2-dev \ + libwebp-dev \ + && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +EXPOSE 8000 + +ENTRYPOINT ["/usr/bin/tini", "--"] + +CMD ["/srv/app/docker-entrypoint.sh"] + +FROM app_python as app_python_debug + +WORKDIR /srv/app + +RUN pip install debugpy==1.6.7 diff --git a/Makefile b/Makefile index e0fc00c5..e6182deb 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,8 @@ vendors: mkdir -p umap/static/umap/vendors/georsstogeojson/ && cp -r node_modules/georsstogeojson/GeoRSSToGeoJSON.js umap/static/umap/vendors/georsstogeojson/ mkdir -p umap/static/umap/vendors/togpx/ && cp -r node_modules/togpx/togpx.js umap/static/umap/vendors/togpx/ mkdir -p umap/static/umap/vendors/tokml && cp -r node_modules/tokml/tokml.js umap/static/umap/vendors/tokml - mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/{dist/L.Control.Locate.css,src/L.Control.Locate.js} umap/static/umap/vendors/locatecontrol/ + mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/dist/L.Control.Locate.css umap/static/umap/vendors/locatecontrol/ + mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/src/L.Control.Locate.js umap/static/umap/vendors/locatecontrol/ installjs: npm install testjsfx: diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 00000000..8147feb0 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,14 @@ +version: '2' + +services: + + app: + build: + target: app_python_debug + environment: + PYTHON_DEBUG: True + volumes: + - ./umap:/srv/app/umap + ports: + - 8000:8000 + - 5678:5678 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..dcb1e2b0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: '2' +services: + postgres: + image: postgis/postgis:14-3.2 + environment: + POSTGRES_PASSWORD: ${DATABASE_PASSWORD} + POSTGRES_USER: ${DATABASE_USER} + volumes: + - ./data:/var/lib/postgresql/data + redis: + image: redis:latest + app: + build: + context: . + target: app_python + environment: + DATABASE_URL: postgis://${DATABASE_USER}:${DATABASE_PASSWORD}@postgres/postgres + REDIS_URL: redis://redis:6379/0 + SECRET_KEY: ${SECRET_KEY} + ALLOWED_HOSTS: ${ALLOWED_HOSTS:-*} + SITE_URL: ${SITE_URL} + LEAFLET_STORAGE_ALLOW_ANONYMOUS: true + depends_on: + - postgres + volumes: + - ./uploads:/srv/app/uploads + ports: + - "8000:8000" diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..9e044821 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -eo pipefail + +# default variables +: "${SLEEP:=1}" +: "${TRIES:=60}" + +function wait_for_database {( + echo "Waiting for database to respond..." + tries=0 + while true; do + [[ $tries -lt $TRIES ]] || return + (echo "from django.db import connection; connection.connect()" | umap shell) >/dev/null + [[ $? -eq 0 ]] && return + sleep $SLEEP + tries=$((tries + 1)) + done +)} + +# first wait for the database +wait_for_database +# then migrate the database +umap migrate +# then collect static files +umap collectstatic --noinput +# create languagae files +#umap storagei18n +# compress static files +umap compress + +if [ "$PYTHON_DEBUG" = true ] ; then + python -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000 --nothreading --noreload +else + exec uwsgi --ini uwsgi.ini +fi diff --git a/package.json b/package.json index e732025c..6e2c3bef 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,6 @@ "mocha": "^2.3.3", "mocha-phantomjs": "^4.0.1", "optimist": "~0.4.0", - "phantomjs": "^1.9.18", "sinon": "^1.10.3", "uglify-js": "~2.2.3" }, diff --git a/requirements-docker.txt b/requirements-docker.txt new file mode 100644 index 00000000..f4b99064 --- /dev/null +++ b/requirements-docker.txt @@ -0,0 +1,3 @@ +django-environ==0.4.1 +django-redis==5.2.0 +uwsgi==2.0.21 diff --git a/umap/settings/dev.py b/umap/settings/dev.py index 9439e43b..327898ab 100644 --- a/umap/settings/dev.py +++ b/umap/settings/dev.py @@ -12,5 +12,6 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'umap', + 'SECRET_KEY': 'some-long-and-weirdly-unrandom-secret-key', } } diff --git a/umap/settings/docker.py b/umap/settings/docker.py new file mode 100644 index 00000000..b405cc7e --- /dev/null +++ b/umap/settings/docker.py @@ -0,0 +1,155 @@ +# -*- coding:utf-8 -*- +""" +Settings for Docker development + +Use this file as a base for your local development settings and copy +it to umap/settings/local.py. It should not be checked into +your code repository. +""" +import environ +from umap.settings.base import * # pylint: disable=W0614,W0401 + +env = environ.Env() + +SECRET_KEY = env('SECRET_KEY') +INTERNAL_IPS = env.list('INTERNAL_IPS', default='127.0.0.1') +ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*']) + +DEBUG = env.bool('PYTHON_DEBUG', default=False) + +ADMIN_EMAILS = env.list('ADMIN_EMAIL', default='') +ADMINS = [(email, email) for email in ADMIN_EMAILS] +MANAGERS = ADMINS + +DATABASES = { + 'default': env.db(default='postgis://localhost:5432/umap') +} + +COMPRESS_ENABLED = True +COMPRESS_OFFLINE = True + +LANGUAGE_CODE = 'en' + +# Set to False if login into django account should not be possible. You can +# administer accounts in the admin interface. +ENABLE_ACCOUNT_LOGIN = env.bool('ENABLE_ACCOUNT_LOGIN', default=True) + +AUTHENTICATION_BACKENDS = () + +# We need email to associate with other Oauth providers +SOCIAL_AUTH_GITHUB_SCOPE = ['user:email'] +SOCIAL_AUTH_GITHUB_KEY = env('GITHUB_KEY', default='') +SOCIAL_AUTH_GITHUB_SECRET = env('GITHUB_SECRET', default='') +if SOCIAL_AUTH_GITHUB_KEY and SOCIAL_AUTH_GITHUB_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.github.GithubOAuth2', + ) +SOCIAL_AUTH_BITBUCKET_KEY = env('BITBUCKET_KEY', default='') +SOCIAL_AUTH_BITBUCKET_SECRET = env('BITBUCKET_SECRET', default='') +if SOCIAL_AUTH_BITBUCKET_KEY and SOCIAL_AUTH_BITBUCKET_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.bitbucket.BitbucketOAuth', + ) + +SOCIAL_AUTH_TWITTER_KEY = env('TWITTER_KEY', default='') +SOCIAL_AUTH_TWITTER_SECRET = env('TWITTER_SECRET', default='') +if SOCIAL_AUTH_TWITTER_KEY and SOCIAL_AUTH_TWITTER_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.twitter.TwitterOAuth', + ) +SOCIAL_AUTH_OPENSTREETMAP_KEY = env('OPENSTREETMAP_KEY', default='') +SOCIAL_AUTH_OPENSTREETMAP_SECRET = env('OPENSTREETMAP_SECRET', default='') +if SOCIAL_AUTH_OPENSTREETMAP_KEY and SOCIAL_AUTH_OPENSTREETMAP_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.openstreetmap.OpenStreetMapOAuth', + ) + +AUTHENTICATION_BACKENDS += ( + 'django.contrib.auth.backends.ModelBackend', +) + +# MIDDLEWARE_CLASSES += ( +# 'social_django.middleware.SocialAuthExceptionMiddleware', +# ) + +SOCIAL_AUTH_RAISE_EXCEPTIONS = False +SOCIAL_AUTH_BACKEND_ERROR_URL = "/" + +# If you want to add a playgroud map, add its primary key +# UMAP_DEMO_PK = 204 +# If you want to add a showcase map on the home page, add its primary key +# UMAP_SHOWCASE_PK = 1156 +# Add a baner to warn people this instance is not production ready. +UMAP_DEMO_SITE = False + +# Whether to allow non authenticated people to create maps. +LEAFLET_STORAGE_ALLOW_ANONYMOUS = env.bool( + 'LEAFLET_STORAGE_ALLOW_ANONYMOUS', + default=False, +) + +# This setting will exclude empty maps (in fact, it will exclude all maps where +# the default center has not been updated) +UMAP_EXCLUDE_DEFAULT_MAPS = False + +# How many maps should be showcased on the main page resp. on the user page +UMAP_MAPS_PER_PAGE = 0 +# How many maps should be showcased on the user page, if owner +UMAP_MAPS_PER_PAGE_OWNER = 10 + +SITE_URL = env('SITE_URL') +SHORT_SITE_URL = env('SHORT_SITE_URL', default=None) + +CACHES = {'default': env.cache('REDIS_URL', default='locmem://')} + +# POSTGIS_VERSION = (2, 1, 0) +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + +# You need to unable accent extension before using UMAP_USE_UNACCENT +# python manage.py dbshell +# CREATE EXTENSION unaccent; +UMAP_USE_UNACCENT = False + +# For static deployment +STATIC_ROOT = '/srv/app/static' + +# For users' statics (geojson mainly) +MEDIA_ROOT = '/srv/app/uploads' + +# Default map location for new maps +LEAFLET_LONGITUDE = env.int('LEAFLET_LONGITUDE', default=2) +LEAFLET_LATITUDE = env.int('LEAFLET_LATITUDE', default=51) +LEAFLET_ZOOM = env.int('LEAFLET_ZOOM', default=6) + +# Number of old version to keep per datalayer. +LEAFLET_STORAGE_KEEP_VERSIONS = env.int( + 'LEAFLET_STORAGE_KEEP_VERSIONS', + default=10, +) + +import sys + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': '[django] %(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'verbose' + }, + }, + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': True, + }, + }, +} diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 00000000..e42c8feb --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,10 @@ +[uwsgi] +http = :$(PORT) +module = umap.wsgi:application +master = True +vacuum = True +max-requests = 5000 +processes = 4 +enable-threads = true +static-map = /static=/srv/app/static +static-map = /uploads=/srv/app/uploads