From 89033e04a2995d3f9c18ba4d2f6dd919a70e6388 Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Fri, 11 Apr 2025 10:09:02 +0200 Subject: [PATCH 1/2] chore: add a deploy overview doc page --- docs/config/settings.md | 6 +++ docs/deploy/nginx.md | 80 ++----------------------------------- docs/deploy/overview.md | 37 ++++++++++++++++++ docs/deploy/wsgi.md | 87 +++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 2 + 5 files changed, 135 insertions(+), 77 deletions(-) create mode 100644 docs/deploy/overview.md create mode 100644 docs/deploy/wsgi.md diff --git a/docs/config/settings.md b/docs/config/settings.md index b31cf3d3..cf180fa5 100644 --- a/docs/config/settings.md +++ b/docs/config/settings.md @@ -95,6 +95,12 @@ A switch will be available for them in the "advanced properties" of the map. See [the documentation about ASGI deployment](../deploy/asgi.md) for more information. +#### REDIS_URL + +Connection URL to the Redis server. Only need for the real-time editing. + +Default: `redis://localhost:6379` + #### SECRET_KEY Must be defined to something unique and secret. diff --git a/docs/deploy/nginx.md b/docs/deploy/nginx.md index c2ec45e4..15335eba 100644 --- a/docs/deploy/nginx.md +++ b/docs/deploy/nginx.md @@ -1,84 +1,10 @@ # Configuring Nginx -Here are some configuration files to use umap with nginx and [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/), a server for python, which will handle your processes for you. +See [WSGI](wsgi.md) or [ASGI](asgi.md) for a basic setup. -```nginx title="nginx.conf" -upstream umap { - server unix:///srv/umap/umap.sock; -} +Then consider adding this configuration -server { - # the port your site will be served on - listen 80; - listen [::]:80; - listen 443 ssl; - listen [::]:443 ssl; - # the domain name it will serve for - server_name your-domain.org; - charset utf-8; - - # max upload size - client_max_body_size 5M; # adjust to taste - - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass umap; - include /srv/umap/uwsgi_params; - } -} -``` - -## uWSGI - - -```nginx title="uwsgi_params" -uwsgi_param QUERY_STRING $query_string; -uwsgi_param REQUEST_METHOD $request_method; -uwsgi_param CONTENT_TYPE $content_type; -uwsgi_param CONTENT_LENGTH $content_length; - -uwsgi_param REQUEST_URI $request_uri; -uwsgi_param PATH_INFO $document_uri; -uwsgi_param DOCUMENT_ROOT $document_root; -uwsgi_param SERVER_PROTOCOL $server_protocol; -uwsgi_param REQUEST_SCHEME $scheme; -uwsgi_param HTTPS $https if_not_empty; - -uwsgi_param REMOTE_ADDR $remote_addr; -uwsgi_param REMOTE_PORT $remote_port; -uwsgi_param SERVER_PORT $server_port; -uwsgi_param SERVER_NAME $server_name; -``` - - -```ini title="uwsgi.ini" -[uwsgi] -uid = umap -gid = users -# Python related settings -# the base directory (full path) -chdir = /srv/umap/ -# umap's wsgi module -module = umap.wsgi -# the virtualenv (full path) -home = /srv/umap/venv - -# process-related settings -# master -master = true -# maximum number of worker processes -processes = 4 -# the socket (use the full path to be safe) -socket = /srv/umap/umap.sock -# ... with appropriate permissions - may be needed -chmod-socket = 666 -stats = /srv/umap/stats.sock -# clear environment on exit -vacuum = true -plugins = python3 -``` - -## Static files +## Static files and geojson ```nginx title="nginx.conf" location /static { diff --git a/docs/deploy/overview.md b/docs/deploy/overview.md new file mode 100644 index 00000000..51091bef --- /dev/null +++ b/docs/deploy/overview.md @@ -0,0 +1,37 @@ +# Deploying uMap + +uMap is a python package, running [Django](https://docs.djangoproject.com/en/5.2/howto/deployment/), +so anyone experimented with this stack will find it familiar, but there are some speficic details +to know about. + +## Data +One important design point of uMap is that while metadata are stored in a PostgreSQL database, the +data itself is stored in the file system, as geojson files. This design choice has been made +to make uMap scale better, as there are much more reads than writes, and when some +map is shared a lot (like on a national media) we want to be able to serve it without needing an +overcomplex and costly stack. + +So when a request for data is made (that is on a *DataLayer*), the flow is that uMap will read +the request headers to check for permissions, and then it will forward the request to Nginx, +that will properly serve the data (a geojson file), without consuming a python worker, and with +much more efficiency than python. + +In DEBUG mode, uMap will serve the geojson itself, but this is not recommended in production, +unless you have a very small audience. + +Data can also be stored in a [S3 like storage](../config/storage/#using-s3). + +## Assets (JS, CSS…) +As any web app, uMap also needs static files to be served. In DEBUG mode, Django will do this +kindly, but not in production. See [Nginx configuration](nginx.md) for this. + +Assets can also be stored in a [S3 like storage](../config/storage/#using-s3). + +## python app (metadata, permissions…) + +uMap needs a python server, which can either be of [WSGI](wsgi.md) or [ASGI](asgi.md) (this later +is needed in order to use the collaborative live editing). + +## Redis + +Still when using the collaborative live editing, uMap needs a [Redis](../config/settings.md#redis_url) server, to act as pubsub. diff --git a/docs/deploy/wsgi.md b/docs/deploy/wsgi.md new file mode 100644 index 00000000..1c632645 --- /dev/null +++ b/docs/deploy/wsgi.md @@ -0,0 +1,87 @@ +# WSGI + +WSGI is the historical standard to serve python in general, and uMap in this case. +From recently, uMap also supports [ASGI](asgi.md), which is required to use the +collaborative editing feature. + +## uWSGI + +In Nginx host, use: + +```nginx title="nginx.conf" +upstream umap { + server unix:///srv/umap/umap.sock; +} + +server { + # the port your site will be served on + listen 80; + listen [::]:80; + listen 443 ssl; + listen [::]:443 ssl; + # the domain name it will serve for + server_name your-domain.org; + charset utf-8; + + # max upload size + client_max_body_size 5M; # adjust to taste + + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass umap; + include /srv/umap/uwsgi_params; + } +} +``` + + + +```nginx title="uwsgi_params" +uwsgi_param QUERY_STRING $query_string; +uwsgi_param REQUEST_METHOD $request_method; +uwsgi_param CONTENT_TYPE $content_type; +uwsgi_param CONTENT_LENGTH $content_length; + +uwsgi_param REQUEST_URI $request_uri; +uwsgi_param PATH_INFO $document_uri; +uwsgi_param DOCUMENT_ROOT $document_root; +uwsgi_param SERVER_PROTOCOL $server_protocol; +uwsgi_param REQUEST_SCHEME $scheme; +uwsgi_param HTTPS $https if_not_empty; + +uwsgi_param REMOTE_ADDR $remote_addr; +uwsgi_param REMOTE_PORT $remote_port; +uwsgi_param SERVER_PORT $server_port; +uwsgi_param SERVER_NAME $server_name; +``` + + +```ini title="uwsgi.ini" +[uwsgi] +uid = umap +gid = users +# Python related settings +# the base directory (full path) +chdir = /srv/umap/ +# umap's wsgi module +module = umap.wsgi +# the virtualenv (full path) +home = /srv/umap/venv + +# process-related settings +# master +master = true +# maximum number of worker processes +processes = 4 +# the socket (use the full path to be safe) +socket = /srv/umap/umap.sock +# ... with appropriate permissions - may be needed +chmod-socket = 666 +stats = /srv/umap/stats.sock +# clear environment on exit +vacuum = true +plugins = python3 +``` + + +See also [Django documentation](https://docs.djangoproject.com/en/5.2/howto/deployment/wsgi/). diff --git a/mkdocs.yml b/mkdocs.yml index c8559383..93ad2a8a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,10 +18,12 @@ nav: - Storage: config/storage.md - Icon packs: config/icons.md - Deployment: + - Overview: deploy/overview.md - Docker: deploy/docker.md - Helm: deploy/helm.md - Nginx: deploy/nginx.md - ASGI: deploy/asgi.md + - WSGI: deploy/wsgi.md - Changelog: changelog.md theme: name: material From 0ec4e74f8f31b6cbb53eddb5750692b2bf3e3c0d Mon Sep 17 00:00:00 2001 From: Yohan Boniface Date: Fri, 11 Apr 2025 10:46:33 +0200 Subject: [PATCH 2/2] wip: add nginx in docker-compose.yml --- Dockerfile | 6 +-- docker-compose.yml | 19 ++++++-- docker/nginx.conf | 111 ++++++++++++++++++++++++++++++++++++++++++ docs/changelog.md | 7 +++ umap/settings/base.py | 4 +- 5 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 docker/nginx.conf diff --git a/Dockerfile b/Dockerfile index d4df0f5c..a35602c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # This part installs deps needed at runtime. -FROM python:3.11-slim AS runtime +FROM python:3.12-slim AS common RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -13,7 +13,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # This part adds deps needed only at buildtime. -FROM runtime AS build +FROM common AS build RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -40,7 +40,7 @@ COPY . /srv/umap RUN /venv/bin/pip install .[docker,s3,sync] -FROM runtime +FROM common COPY --from=build /srv/umap/docker/ /srv/umap/docker/ COPY --from=build /venv/ /venv/ diff --git a/docker-compose.yml b/docker-compose.yml index ba958081..e66a7a8a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,10 +26,10 @@ services: condition: service_healthy redis: condition: service_healthy - image: umap/umap:3.0.0 - ports: - - "${PORT-8000}:8000" + image: umap/umap:3.0.2 environment: + - STATIC_ROOT=/srv/umap/static + - MEDIA_ROOT=/srv/umap/uploads - DATABASE_URL=postgis://postgres@db/postgres - SECRET_KEY=some-long-and-weirdly-unrandom-secret-key - SITE_URL=https://umap.local/ @@ -39,7 +39,20 @@ services: - REDIS_URL=redis://redis:6379 volumes: - data:/srv/umap/uploads + - static:/srv/umap/static + + proxy: + image: nginx:latest + ports: + - "8000:80" + volumes: + - ./docker/nginx.conf:/etc/nginx/nginx.conf:ro + - static:/static:ro + - data:/data:ro + depends_on: + - app volumes: data: + static: db: diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 00000000..fe73d2a6 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,111 @@ +events { + worker_connections 1024; # Adjust this to your needs +} + + + +http { + proxy_cache_path /tmp/nginx_ajax_proxy_cache levels=1:2 keys_zone=ajax_proxy:10m inactive=60m; + proxy_cache_key "$uri$is_args$args"; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + types { + application/javascript mjs; + } + + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + # Server block + server { + listen 80; + server_name localhost; + + # Static file serving + location /static/ { + alias /static/; + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css; + expires 365d; + access_log /dev/null; + } + + # Geojson files + location /uploads/ { + alias /data/; + expires 30d; + } + + location /favicon.ico { + alias /static/favicon.ico; + } + + # X-Accel-Redirect + location /internal/ { + internal; + gzip_vary on; + gzip_static on; + add_header X-DataLayer-Version $upstream_http_x_datalayer_version; + alias /data/; + } + + # Ajax proxy + location ~ ^/proxy/(.*) { + internal; + add_header X-Proxy-Cache $upstream_cache_status always; + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache ajax_proxy; + proxy_cache_valid 1m; # Default. Umap will override using X-Accel-Expires + set $target_url $1; + # URL is encoded, so we need a few hack to clean it back. + if ( $target_url ~ (.+)%3A%2F%2F(.+) ){ # fix :// between scheme and destination + set $target_url $1://$2; + } + if ( $target_url ~ (.+?)%3A(.*) ){ # fix : between destination and port + set $target_url $1:$2; + } + if ( $target_url ~ (.+?)%2F(.*) ){ # fix / after port, the rest will be decoded by proxy_pass + set $target_url $1/$2; + } + resolver 8.8.8.8; + add_header X-Proxy-Target $target_url; # For debugging + proxy_pass_request_headers off; + proxy_set_header Content-Type $http_content_type; + proxy_set_header Content-Encoding $http_content_encoding; + proxy_set_header Content-Length $http_content_length; + proxy_read_timeout 10s; + proxy_connect_timeout 5s; + proxy_ssl_server_name on; + proxy_pass $target_url; + proxy_intercept_errors on; + error_page 301 302 307 = @handle_proxy_redirect; + } + location @handle_proxy_redirect { + resolver 8.8.8.8; + set $saved_redirect_location '$upstream_http_location'; + proxy_pass $saved_redirect_location; + } + + # Proxy pass to ASGI server + location / { + proxy_pass http://app:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_redirect off; + proxy_buffering off; + } + } +} diff --git a/docs/changelog.md b/docs/changelog.md index ef6e7f3f..5aca981b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -32,6 +32,13 @@ Other notable changes: Note: you may want to update your search index to include the category search, see https://docs.umap-project.org/en/stable/config/settings/#umap_search_configuration + +### Breaking change + +* The Docker image will not serve assets and data files anymore, an Nginx container must + be configured. See [docker-compose.yml](https://github.com/umap-project/umap/blob/master/docker-compose.yml) + for an example. + ### New features * add collaborative real-time map editing * add atomic undo redo by @yohanboniface in #2570 diff --git a/umap/settings/base.py b/umap/settings/base.py index 5faf21c1..6edb7fb4 100644 --- a/umap/settings/base.py +++ b/umap/settings/base.py @@ -164,8 +164,8 @@ LOGIN_REDIRECT_URL = "login_popup_end" STATIC_URL = "/static/" MEDIA_URL = "/uploads/" -STATIC_ROOT = os.path.join("static") -MEDIA_ROOT = os.path.join("uploads") +STATIC_ROOT = env("STATIC_ROOT", default=os.path.join("static")) +MEDIA_ROOT = env("MEDIA_ROOT", default=os.path.join("uploads")) STATICFILES_FINDERS = [ "django.contrib.staticfiles.finders.FileSystemFinder",