document that nginx needs to be added in Docker stack to serve statics (#2636)

Dockerfile

@@ -1,5 +1,5 @@
 # This part installs deps needed at runtime.
-FROM python:3.11-slim AS runtime
+FROM python:3.12-slim AS common
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -13,7 +13,7 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # This part adds deps needed only at buildtime.
-FROM runtime AS build
+FROM common AS build
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -40,7 +40,7 @@ COPY . /srv/umap
 
 RUN /venv/bin/pip install .[docker,s3,sync]
 
-FROM runtime
+FROM common
 
 COPY --from=build /srv/umap/docker/ /srv/umap/docker/
 COPY --from=build /venv/ /venv/

docker-compose.yml

@@ -26,10 +26,10 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
-    image: umap/umap:3.0.0
-    ports:
-      - "${PORT-8000}:8000"
+    image: umap/umap:3.0.2
     environment:
+      - STATIC_ROOT=/srv/umap/static
+      - MEDIA_ROOT=/srv/umap/uploads
       - DATABASE_URL=postgis://postgres@db/postgres
       - SECRET_KEY=some-long-and-weirdly-unrandom-secret-key
       - SITE_URL=https://umap.local/
@@ -39,7 +39,20 @@ services:
       - REDIS_URL=redis://redis:6379
     volumes:
       - data:/srv/umap/uploads
+      - static:/srv/umap/static
+
+  proxy:
+    image: nginx:latest
+    ports:
+      - "8000:80"
+    volumes:
+      - ./docker/nginx.conf:/etc/nginx/nginx.conf:ro
+      - static:/static:ro
+      - data:/data:ro
+    depends_on:
+      - app
 
 volumes:
   data:
+  static:
   db:

docker/nginx.conf

@@ -0,0 +1,111 @@
+events {
+    worker_connections  1024;  # Adjust this to your needs
+}
+
+
+
+http {
+    proxy_cache_path /tmp/nginx_ajax_proxy_cache levels=1:2 keys_zone=ajax_proxy:10m inactive=60m;
+    proxy_cache_key "$uri$is_args$args";
+
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+    }
+
+    types {
+      application/javascript mjs;
+    }
+
+    include       mime.types;
+    default_type  application/octet-stream;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # Server block
+    server {
+        listen       80;
+        server_name  localhost;
+
+        # Static file serving
+        location /static/ {
+            alias /static/;
+            gzip on;
+            gzip_vary on;
+            gzip_proxied any;
+            gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css;
+            expires 365d;
+            access_log /dev/null;
+        }
+
+        # Geojson files
+        location /uploads/  {
+            alias /data/;
+            expires 30d;
+        }
+
+        location /favicon.ico {
+            alias /static/favicon.ico;
+        }
+
+        # X-Accel-Redirect
+        location /internal/ {
+            internal;
+            gzip_vary on;
+            gzip_static on;
+            add_header X-DataLayer-Version $upstream_http_x_datalayer_version;
+            alias /data/;
+        }
+
+        # Ajax proxy
+        location ~ ^/proxy/(.*) {
+            internal;
+            add_header X-Proxy-Cache $upstream_cache_status always;
+            proxy_cache_background_update on;
+            proxy_cache_use_stale updating;
+            proxy_cache ajax_proxy;
+            proxy_cache_valid 1m;  # Default. Umap will override using X-Accel-Expires
+            set $target_url $1;
+            # URL is encoded, so we need a few hack to clean it back.
+            if ( $target_url ~ (.+)%3A%2F%2F(.+) ){ # fix :// between scheme and destination
+              set $target_url $1://$2;
+            }
+            if ( $target_url ~ (.+?)%3A(.*) ){ # fix : between destination and port
+              set $target_url $1:$2;
+            }
+            if ( $target_url ~ (.+?)%2F(.*) ){ # fix / after port, the rest will be decoded by proxy_pass
+              set $target_url $1/$2;
+            }
+            resolver 8.8.8.8;
+            add_header X-Proxy-Target $target_url; # For debugging
+            proxy_pass_request_headers off;
+            proxy_set_header Content-Type $http_content_type;
+            proxy_set_header Content-Encoding $http_content_encoding;
+            proxy_set_header Content-Length $http_content_length;
+            proxy_read_timeout 10s;
+            proxy_connect_timeout 5s;
+            proxy_ssl_server_name on;
+            proxy_pass $target_url;
+            proxy_intercept_errors on;
+            error_page 301 302 307 = @handle_proxy_redirect;
+        }
+        location @handle_proxy_redirect {
+            resolver 8.8.8.8;
+            set $saved_redirect_location '$upstream_http_location';
+            proxy_pass $saved_redirect_location;
+        }
+
+        # Proxy pass to ASGI server
+        location / {
+            proxy_pass http://app:8000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_redirect off;
+            proxy_buffering off;
+        }
+    }
+}

docs/changelog.md

@@ -32,6 +32,13 @@ Other notable changes:
 Note: you may want to update your search index to include the category search,
 see https://docs.umap-project.org/en/stable/config/settings/#umap_search_configuration
 
+
+### Breaking change
+
+* The Docker image will not serve assets and data files anymore, an Nginx container must
+  be configured. See [docker-compose.yml](https://github.com/umap-project/umap/blob/master/docker-compose.yml)
+  for an example.
+
 ### New features
 * add collaborative real-time map editing
 * add atomic undo redo by @yohanboniface in #2570

docs/config/settings.md

@@ -95,6 +95,12 @@ A switch will be available for them in the "advanced properties" of the map.
 
 See [the documentation about ASGI deployment](../deploy/asgi.md) for more information.
 
+#### REDIS_URL
+
+Connection URL to the Redis server. Only need for the real-time editing.
+
+Default: `redis://localhost:6379`
+
 #### SECRET_KEY
 
 Must be defined to something unique and secret.

docs/deploy/nginx.md

@@ -1,84 +1,10 @@
 # Configuring Nginx
 
-Here are some configuration files to use umap with nginx and [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/), a server for python, which will handle your processes for you.
+See [WSGI](wsgi.md) or [ASGI](asgi.md) for a basic setup.
 
-```nginx title="nginx.conf"
-upstream umap {
-    server unix:///srv/umap/umap.sock;
-}
+Then consider adding this configuration
 
-server {
-    # the port your site will be served on
-    listen      80;
-    listen   [::]:80;
-    listen      443 ssl;
-    listen   [::]:443 ssl;
-    # the domain name it will serve for
-    server_name your-domain.org;
-    charset     utf-8;
-
-    # max upload size
-    client_max_body_size 5M;   # adjust to taste
-
-    # Finally, send all non-media requests to the Django server.
-    location / {
-        uwsgi_pass  umap;
-        include /srv/umap/uwsgi_params;
-    }
-}
-```
-
-## uWSGI
-
-
-```nginx title="uwsgi_params"
-uwsgi_param  QUERY_STRING       $query_string;
-uwsgi_param  REQUEST_METHOD     $request_method;
-uwsgi_param  CONTENT_TYPE       $content_type;
-uwsgi_param  CONTENT_LENGTH     $content_length;
-
-uwsgi_param  REQUEST_URI        $request_uri;
-uwsgi_param  PATH_INFO          $document_uri;
-uwsgi_param  DOCUMENT_ROOT      $document_root;
-uwsgi_param  SERVER_PROTOCOL    $server_protocol;
-uwsgi_param  REQUEST_SCHEME     $scheme;
-uwsgi_param  HTTPS              $https if_not_empty;
-
-uwsgi_param  REMOTE_ADDR        $remote_addr;
-uwsgi_param  REMOTE_PORT        $remote_port;
-uwsgi_param  SERVER_PORT        $server_port;
-uwsgi_param  SERVER_NAME        $server_name;
-```
-
-
-```ini title="uwsgi.ini"
-[uwsgi]
-uid = umap
-gid = users
-# Python related settings
-# the base directory (full path)
-chdir = /srv/umap/
-# umap's wsgi module
-module = umap.wsgi
-# the virtualenv (full path)
-home = /srv/umap/venv
-
-# process-related settings
-# master
-master = true
-# maximum number of worker processes
-processes = 4
-# the socket (use the full path to be safe)
-socket = /srv/umap/umap.sock
-# ... with appropriate permissions - may be needed
-chmod-socket = 666
-stats = /srv/umap/stats.sock
-# clear environment on exit
-vacuum = true
-plugins = python3
-```
-
-## Static files
+## Static files and geojson
 
 ```nginx title="nginx.conf"
 location /static {

docs/deploy/overview.md

@@ -0,0 +1,37 @@
+# Deploying uMap
+
+uMap is a python package, running [Django](https://docs.djangoproject.com/en/5.2/howto/deployment/),
+so anyone experimented with this stack will find it familiar, but there are some speficic details
+to know about.
+
+## Data
+One important design point of uMap is that while metadata are stored in a PostgreSQL database, the
+data itself is stored in the file system, as geojson files. This design choice has been made
+to make uMap scale better, as there are much more reads than writes, and when some
+map is shared a lot (like on a national media) we want to be able to serve it without needing an
+overcomplex and costly stack.
+
+So when a request for data is made (that is on a *DataLayer*), the flow is that uMap will read
+the request headers to check for permissions, and then it will forward the request to Nginx,
+that will properly serve the data (a geojson file), without consuming a python worker, and with
+much more efficiency than python.
+
+In DEBUG mode, uMap will serve the geojson itself, but this is not recommended in production,
+unless you have a very small audience.
+
+Data can also be stored in a [S3 like storage](../config/storage/#using-s3).
+
+## Assets (JS, CSS…)
+As any web app, uMap also needs static files to be served. In DEBUG mode, Django will do this
+kindly, but not in production. See [Nginx configuration](nginx.md) for this.
+
+Assets can also be stored in a [S3 like storage](../config/storage/#using-s3).
+
+## python app (metadata, permissions…)
+
+uMap needs a python server, which can either be of [WSGI](wsgi.md) or [ASGI](asgi.md) (this later
+is needed in order to use the collaborative live editing).
+
+## Redis
+
+Still when using the collaborative live editing, uMap needs a [Redis](../config/settings.md#redis_url) server, to act as pubsub.

docs/deploy/wsgi.md

@@ -0,0 +1,87 @@
+# WSGI
+
+WSGI is the historical standard to serve python in general, and uMap in this case.
+From recently, uMap also supports [ASGI](asgi.md), which is required to use the
+collaborative editing feature.
+
+## uWSGI
+
+In Nginx host, use:
+
+```nginx title="nginx.conf"
+upstream umap {
+    server unix:///srv/umap/umap.sock;
+}
+
+server {
+    # the port your site will be served on
+    listen      80;
+    listen   [::]:80;
+    listen      443 ssl;
+    listen   [::]:443 ssl;
+    # the domain name it will serve for
+    server_name your-domain.org;
+    charset     utf-8;
+
+    # max upload size
+    client_max_body_size 5M;   # adjust to taste
+
+    # Finally, send all non-media requests to the Django server.
+    location / {
+        uwsgi_pass  umap;
+        include /srv/umap/uwsgi_params;
+    }
+}
+```
+
+
+
+```nginx title="uwsgi_params"
+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  REQUEST_SCHEME     $scheme;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;
+```
+
+
+```ini title="uwsgi.ini"
+[uwsgi]
+uid = umap
+gid = users
+# Python related settings
+# the base directory (full path)
+chdir = /srv/umap/
+# umap's wsgi module
+module = umap.wsgi
+# the virtualenv (full path)
+home = /srv/umap/venv
+
+# process-related settings
+# master
+master = true
+# maximum number of worker processes
+processes = 4
+# the socket (use the full path to be safe)
+socket = /srv/umap/umap.sock
+# ... with appropriate permissions - may be needed
+chmod-socket = 666
+stats = /srv/umap/stats.sock
+# clear environment on exit
+vacuum = true
+plugins = python3
+```
+
+
+See also [Django documentation](https://docs.djangoproject.com/en/5.2/howto/deployment/wsgi/).

mkdocs.yml

@@ -18,10 +18,12 @@ nav:
       - Storage: config/storage.md
       - Icon packs: config/icons.md
   - Deployment:
+      - Overview: deploy/overview.md
       - Docker: deploy/docker.md
       - Helm: deploy/helm.md
       - Nginx: deploy/nginx.md
       - ASGI: deploy/asgi.md
+      - WSGI: deploy/wsgi.md
   - Changelog: changelog.md
 theme:
   name: material

umap/settings/base.py

@@ -164,8 +164,8 @@ LOGIN_REDIRECT_URL = "login_popup_end"
 STATIC_URL = "/static/"
 MEDIA_URL = "/uploads/"
 
-STATIC_ROOT = os.path.join("static")
-MEDIA_ROOT = os.path.join("uploads")
+STATIC_ROOT = env("STATIC_ROOT", default=os.path.join("static"))
+MEDIA_ROOT = env("MEDIA_ROOT", default=os.path.join("uploads"))
 
 STATICFILES_FINDERS = [
     "django.contrib.staticfiles.finders.FileSystemFinder",