From 47bcacaa622e5a083b88876dc6a76a03187fa0da Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 20 Feb 2017 16:32:41 +0100 Subject: [PATCH] Added Docker setup. This adds: - a Dockerfile - a Docker compose file for easy testing - a Travis CI setup - so that it can build a Docker image and push to Docker Hub automatically - it does that on every Git tag as well and push a equally tagged version to Docker Hub - extends the Makefile to add some helper tasks for docker (e.g. make docker-test) --- .gitignore | 3 + .travis.yml | 21 +++++++ Dockerfile | 55 +++++++++++++++++ Makefile | 14 +++++ docker-compose.yml | 22 +++++++ docker-entrypoint.sh | 31 ++++++++++ requirements-docker.txt | 3 + setup.py | 13 +++- umap/settings/docker.py | 128 ++++++++++++++++++++++++++++++++++++++++ uwsgi.ini | 10 ++++ 10 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 .travis.yml create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh create mode 100644 requirements-docker.txt create mode 100644 umap/settings/docker.py create mode 100644 uwsgi.ini diff --git a/.gitignore b/.gitignore index 60d85dc4..77ad1c98 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ docs/_build umap/remote_static .idea tmp/* +/static/ +.bash_history +.python_history ### Python ### # Byte-compiled / optimized / DLL files diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..8ab3f8b9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +sudo: required + +language: python + +python: 3.5 + +services: + - docker + +install: + - pip install -e .[dev] + +script: + - make test + +after_success: + - export DOCKER_IMAGE_NAME=umap-project/umap + - docker build -t $DOCKER_IMAGE_NAME:latest . + - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; + - if [ ! -z "$TRAVIS_TAG" ]; then docker tag $DOCKER_IMAGE_NAME:latest $DOCKER_IMAGE_NAME:$TRAVIS_TAG; fi + - docker push $DOCKER_IMAGE_NAME; diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..16615039 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +FROM python:3.5 + +ENV PYTHONUNBUFFERED=1 \ + UMAP_SETTINGS=/srv/umap/umap/settings/docker.py \ + PORT=8000 + +# create a user account and group to run uMap +RUN mkdir -p /srv/umap/{data,uploads} && \ + chown -R 10001:10001 /srv/umap && \ + groupadd --gid 10001 umap && \ + useradd --no-create-home --uid 10001 --gid 10001 --home-dir /srv/umap umap + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + binutils \ + libproj-dev \ + gdal-bin \ + build-essential \ + curl \ + git \ + libpq-dev \ + postgresql-client \ + gettext \ + sqlite3 \ + libffi-dev \ + libtiff5-dev \ + libjpeg62-turbo-dev \ + zlib1g-dev \ + libfreetype6-dev \ + liblcms2-dev \ + libwebp-dev \ + && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Add Tini +ENV TINI_VERSION v0.14.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini + +COPY . /srv/umap + +WORKDIR /srv/umap + +RUN pip install --no-cache .[docker] + +USER umap + +EXPOSE 8000 + +ENTRYPOINT ["/tini", "--"] + +CMD ["/srv/umap/docker-entrypoint.sh"] diff --git a/Makefile b/Makefile index 09c76e57..c2937a6c 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,16 @@ test: py.test + +docker-build: + docker-compose build + +docker-up: + docker-compose up + +docker-stop: + docker-compose stop + +docker-test: + docker-compose run app make test + +.PHONY: test docker-build docker-up docker-stop docker-test diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..70544fea --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +version: '2' +services: + db: + image: mdillon/postgis:9.6 + redis: + image: redis:latest + app: + build: . + environment: + - DATABASE_URL=postgis://postgres@db/postgres + - REDIS_URL=redis://redis:6379/0 + - SECRET_KEY=some-long-and-weirdly-unrandom-secret-key + - ALLOWED_HOSTS=* + - SITE_URL=http://localhost:8000/ + - LEAFLET_STORAGE_ALLOW_ANONYMOUS=True + volumes: + - $PWD:/srv/umap + ports: + - "8000:8000" + links: + - db + - redis diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..fc926ab5 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -eo pipefail + +# default variables +: "${SLEEP:=1}" +: "${TRIES:=60}" + +function wait_for_database {( + echo "Waiting for database to respond..." + tries=0 + while true; do + [[ $tries -lt $TRIES ]] || return + (echo "from django.db import connection; connection.connect()" | umap shell) >/dev/null 2>&1 + [[ $? -eq 0 ]] && return + sleep $SLEEP + tries=$((tries + 1)) + done +)} + +# first wait for the database +wait_for_database +# then migrate the database +umap migrate +# then collect static files +umap collectstatic --noinput +# create languagae files +umap storagei18n +# compress static files +umap compress +# run uWSGI +exec uwsgi --ini uwsgi.ini diff --git a/requirements-docker.txt b/requirements-docker.txt new file mode 100644 index 00000000..b1955e87 --- /dev/null +++ b/requirements-docker.txt @@ -0,0 +1,3 @@ +django-environ==0.4.1 +django-redis==4.7.0 +uwsgi==2.0.14 diff --git a/setup.py b/setup.py index f998b162..65520ce3 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,11 @@ long_description = codecs.open('README.md', "r", "utf-8").read() def is_pkg(line): return line and not line.startswith(('--', 'git', '#')) -with io.open('requirements.txt', encoding='utf-8') as reqs: - install_requires = [l for l in reqs.read().split('\n') if is_pkg(l)] + +def reqs_to_list(filename): + with io.open('requirements.txt', encoding='utf-8') as reqs: + return [l for l in reqs.read().split('\n') if is_pkg(l)] + setup( name="umap-project", @@ -30,7 +33,11 @@ setup( platforms=["any"], zip_safe=True, long_description=long_description, - install_requires=install_requires, + install_requires=reqs_to_list('requirements.txt'), + extras_require={ + 'dev': reqs_to_list('requirements-dev.txt'), + 'docker': reqs_to_list('requirements-docker.txt'), + }, classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", diff --git a/umap/settings/docker.py b/umap/settings/docker.py new file mode 100644 index 00000000..fea7cff7 --- /dev/null +++ b/umap/settings/docker.py @@ -0,0 +1,128 @@ +# -*- coding:utf-8 -*- +""" +Settings for Docker development + +Use this file as a base for your local development settings and copy +it to umap/settings/local.py. It should not be checked into +your code repository. +""" +import environ +from umap.settings.base import * # pylint: disable=W0614,W0401 + +env = environ.Env() + +SECRET_KEY = env('SECRET_KEY') +INTERNAL_IPS = env.list('INTERNAL_IPS', default='127.0.0.1') +ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default='*') + +DEBUG = env.bool('DEBUG', default=False) + +ADMIN_EMAILS = env.list('ADMIN_EMAIL', default='') +ADMINS = [(email, email) for email in ADMIN_EMAILS] +MANAGERS = ADMINS + +DATABASES = { + 'default': env.db(default='postgis://localhost:5432/umap') +} + +COMPRESS_ENABLED = True +COMPRESS_OFFLINE = True + +LANGUAGE_CODE = 'en' + +# Set to False if login into django account should not be possible. You can +# administer accounts in the admin interface. +ENABLE_ACCOUNT_LOGIN = env.bool('ENABLE_ACCOUNT_LOGIN', default=True) + +AUTHENTICATION_BACKENDS = () + +# We need email to associate with other Oauth providers +SOCIAL_AUTH_GITHUB_SCOPE = ['user:email'] +SOCIAL_AUTH_GITHUB_KEY = env('GITHUB_KEY', default='') +SOCIAL_AUTH_GITHUB_SECRET = env('GITHUB_SECRET', default='') +if SOCIAL_AUTH_GITHUB_KEY and SOCIAL_AUTH_GITHUB_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.github.GithubOAuth2', + ) +SOCIAL_AUTH_BITBUCKET_KEY = env('BITBUCKET_KEY', default='') +SOCIAL_AUTH_BITBUCKET_SECRET = env('BITBUCKET_SECRET', default='') +if SOCIAL_AUTH_BITBUCKET_KEY and SOCIAL_AUTH_BITBUCKET_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.bitbucket.BitbucketOAuth', + ) + +SOCIAL_AUTH_TWITTER_KEY = env('TWITTER_KEY', default='') +SOCIAL_AUTH_TWITTER_SECRET = env('TWITTER_SECRET', default='') +if SOCIAL_AUTH_TWITTER_KEY and SOCIAL_AUTH_TWITTER_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.twitter.TwitterOAuth', + ) +SOCIAL_AUTH_OPENSTREETMAP_KEY = env('OPENSTREETMAP_KEY', default='') +SOCIAL_AUTH_OPENSTREETMAP_SECRET = env('OPENSTREETMAP_SECRET', default='') +if SOCIAL_AUTH_OPENSTREETMAP_KEY and SOCIAL_AUTH_OPENSTREETMAP_SECRET: + AUTHENTICATION_BACKENDS += ( + 'social_core.backends.openstreetmap.OpenStreetMapOAuth', + ) + +AUTHENTICATION_BACKENDS += ( + 'django.contrib.auth.backends.ModelBackend', +) + +MIDDLEWARE_CLASSES += ( + 'social_django.middleware.SocialAuthExceptionMiddleware', +) + +SOCIAL_AUTH_RAISE_EXCEPTIONS = False +SOCIAL_AUTH_BACKEND_ERROR_URL = "/" + +# If you want to add a playgroud map, add its primary key +# UMAP_DEMO_PK = 204 +# If you want to add a showcase map on the home page, add its primary key +# UMAP_SHOWCASE_PK = 1156 +# Add a baner to warn people this instance is not production ready. +UMAP_DEMO_SITE = False + +# Whether to allow non authenticated people to create maps. +LEAFLET_STORAGE_ALLOW_ANONYMOUS = env.bool( + 'LEAFLET_STORAGE_ALLOW_ANONYMOUS', + default=False, +) + +# This setting will exclude empty maps (in fact, it will exclude all maps where +# the default center has not been updated) +UMAP_EXCLUDE_DEFAULT_MAPS = False + +# How many maps should be showcased on the main page resp. on the user page +UMAP_MAPS_PER_PAGE = 5 +# How many maps should be showcased on the user page, if owner +UMAP_MAPS_PER_PAGE_OWNER = 10 + +SITE_URL = env('SITE_URL') +SHORT_SITE_URL = env('SHORT_SITE_URL', default=None) + +CACHES = {'default': env.cache('REDIS_URL', default='locmem://')} + +# POSTGIS_VERSION = (2, 1, 0) +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + +# You need to unable accent extension before using UMAP_USE_UNACCENT +# python manage.py dbshell +# CREATE EXTENSION unaccent; +UMAP_USE_UNACCENT = False + +# For static deployment +STATIC_ROOT = '/srv/umap/static' + +# For users' statics (geojson mainly) +MEDIA_ROOT = '/srv/umap/uploads' + +# Default map location for new maps +LEAFLET_LONGITUDE = env.int('LEAFLET_LONGITUDE', default=2) +LEAFLET_LATITUDE = env.int('LEAFLET_LATITUDE', default=51) +LEAFLET_ZOOM = env.int('LEAFLET_ZOOM', default=6) + +# Number of old version to keep per datalayer. +LEAFLET_STORAGE_KEEP_VERSIONS = env.int( + 'LEAFLET_STORAGE_KEEP_VERSIONS', + default=10, +) diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 00000000..2cf2c279 --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,10 @@ +[uwsgi] +http = :$(PORT) +module = umap.wsgi:application +master = True +vacuum = True +max-requests = 5000 +processes = 4 +enable-threads = true +static-map = /static=/srv/umap/static +static-map = /uploads=/srv/umap/uploads