Add support for running Redis inside the pretalx container

Dockerfile

@@ -37,6 +37,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
         gettext locales \
         libmariadb3 libmemcached11t64 \
         nodejs npm \
+        redis-server \
         supervisor && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \

README.md

@@ -35,6 +35,10 @@ outdated. Please make sure you read the files before executing them, and check t
   * `GUNICORN_MAX_REQUESTS` and `GUNICORN_MAX_REQUESTS_JITTER` to configure the requests a worker instance will process before restarting.
   * `GUNICORN_FORWARDED_ALLOW_IPS` lets you specify which IPs to trust (i.e. which reverse proxies' `X-Forwarded-*` headers can be used to infer connection security).
   * `GUNICORN_BIND_ADDR` can be used to change the interface and port that Gunicorn will listen on. Default: `0.0.0.0:80`
+* Optional: Set `REDIS_BUNDLED` to `yes` to run Redis inside the pretalx container, instead of as a
+  separate service. pretalx is pointed at it for you, unless you set `PRETALX_REDIS` or
+  `PRETALX_CELERY_*` yourself. Its data is kept under `$PRETALX_DATA_DIR/redis`, and `REDIS_ARGS`
+  passes options on to `redis-server`, such as `--appendonly yes`.
 
   Here's how to set an environment variable [in
   `docker-compose.yml`](https://docs.docker.com/compose/environment-variables/set-environment-variables/)

deployment/docker/pretalx.bash

@@ -16,6 +16,22 @@ GUNICORN_BIND_ADDR="${GUNICORN_BIND_ADDR:-0.0.0.0:80}"
 AUTOMIGRATE="${AUTOMIGRATE:-yes}"
 AUTOREBUILD="${AUTOREBUILD:-yes}"
 
+REDIS_BUNDLED="${REDIS_BUNDLED:-no}"
+
+# Point pretalx at the Redis bundled in this image. Environment variables are
+# the last config layer pretalx reads, so these take precedence over any
+# [redis]/[celery] settings in pretalx.cfg. REDIS_AUTOSTART is what
+# supervisord.conf reads to decide whether to run its redis program.
+if [ "$REDIS_BUNDLED" = "yes" ]; then
+    export REDIS_AUTOSTART=true
+    export PRETALX_REDIS="${PRETALX_REDIS:-redis://127.0.0.1:6379/0}"
+    export PRETALX_REDIS_SESSIONS="${PRETALX_REDIS_SESSIONS:-True}"
+    export PRETALX_CELERY_BACKEND="${PRETALX_CELERY_BACKEND:-redis://127.0.0.1:6379/1}"
+    export PRETALX_CELERY_BROKER="${PRETALX_CELERY_BROKER:-redis://127.0.0.1:6379/2}"
+else
+    export REDIS_AUTOSTART=false
+fi
+
 if [ "$PRETALX_FILESYSTEM_LOGS" != "/data/logs" ]; then
     export PRETALX_FILESYSTEM_LOGS
 fi
@@ -39,18 +55,47 @@ if [ "$PRETALX_FILESYSTEM_STATIC" != "/pretalx/src/static.dist" ] &&
     flock --nonblock /pretalx/.lockfile python3 -m pretalx rebuild
 fi
 
+if [ "$1" == "redis" ]; then
+    mkdir -p "$PRETALX_DATA_DIR/redis"
+    # Bind to loopback only: this Redis serves the processes in this container
+    # and runs without authentication.
+    # shellcheck disable=SC2086
+    exec redis-server --bind 127.0.0.1 --port 6379 \
+        --dir "$PRETALX_DATA_DIR/redis" $REDIS_ARGS
+fi
+
+# "all" runs nothing itself: Redis, the web worker and the task worker are all
+# supervisord programs, and the migrations happen in the web worker below.
+if [ "$1" == "all" ]; then
+    exec /usr/bin/supervisord -n -c /etc/supervisord.conf
+fi
+
+# pretalx runs Django system checks that reach for the cache and the celery
+# broker, so everything below needs Redis to be up. supervisord runs it as a
+# program of its own, but starts all of them in parallel, so wait for it.
+if [ "$REDIS_BUNDLED" = "yes" ]; then
+    for attempt in $(seq 1 120); do
+        redis-cli -h 127.0.0.1 -p 6379 ping >/dev/null 2>&1 && break
+        if [ "$attempt" = 120 ]; then
+            echo "Timed out waiting for the bundled Redis" >&2
+            exit 1
+        fi
+        sleep 0.5
+    done
+fi
+
 if [ "$1" == "cron" ]; then
     exec python3 -m pretalx runperiodic
 fi
 
-if [ "$AUTOMIGRATE" = "yes" ]; then
+# Only the web worker migrates, so that the two workers supervisord starts in
+# parallel cannot run migrations against each other. The task worker only
+# needs the broker to start up, and picks up tasks once the web worker is
+# serving anyway.
+if [ "$AUTOMIGRATE" = "yes" ] && [ "$1" != "taskworker" ]; then
     python3 -m pretalx migrate --noinput
 fi
 
-if [ "$1" == "all" ]; then
-    exec /usr/bin/supervisord -n -c /etc/supervisord.conf
-fi
-
 if [ "$1" == "webworker" ]; then
     exec gunicorn pretalx.wsgi \
         --name pretalx \

deployment/docker/supervisord.conf

@@ -17,6 +17,20 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 [supervisorctl]
 serverurl=unix:///tmp/supervisor.sock
 
+; Only runs when REDIS_BUNDLED is set to "yes"; deployments using an external
+; Redis leave it stopped. The entrypoint normalises REDIS_BUNDLED to
+; true/false for this.
+[program:redis]
+command=/usr/local/bin/pretalx redis
+autostart=%(ENV_REDIS_AUTOSTART)s
+autorestart=true
+priority=1
+user=pretalxuser
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
 [program:pretalxweb]
 command=/usr/local/bin/pretalx webworker
 autostart=true

docker-compose.yml

@@ -14,6 +14,9 @@ services:
       # Hint: Make sure you serve all requests for the `/static/` and `/media/` paths when debug is False. See [installation](https://docs.pretalx.org/administrator/installation/#step-7-ssl) for more information
       PRETALX_FILESYSTEM_MEDIA: /public/media
       PRETALX_FILESYSTEM_STATIC: /public/static
+      # To run Redis in this container, drop the `redis` service below and its
+      # `depends_on` entry above, and see the README.
+      # REDIS_BUNDLED: "yes"
     ports:
       - "80:80"
     volumes: