diff --git a/docker/Dockerfile b/docker/Dockerfile index 0c4901c865..649ae6e1eb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,6 +23,10 @@ ARG NGINX_VERSION=1.29.5 ARG NGINX_SHA256=1d13701a5f9f3fb01aaa88cef2344d65b6b5bf6b7d9fa4cf0dca557a8d7702ba ARG UV_VERSION=0.11.2 ARG UV_SHA256=db7642df9c7e6214d4d7df81cfc3e8327768dd15565b1eb414bb83004f64d463 +ARG S6_OVERLAY_VERSION=3.2.3.2 +ARG S6_OVERLAY_NOARCH_SHA256=5379750ed30a84bbd2e2dd74847ba6b5bd29cd0b2e3ea2ec58049b57eb2eda12 +ARG S6_OVERLAY_X86_64_SHA256=e6befcc96a437a3831386ecfc51808c5d3e939dc5fe3c02ae9284599e8aa2408 +ARG S6_OVERLAY_AARCH64_SHA256=b17f17a82e7a515c682a91edaf2ffdabb73f891981b6c1fd712115693a2f8b4c FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}@sha256:${PYTHON_ALPINE_SHA256} AS python-alias @@ -197,8 +201,33 @@ RUN mkdir -p ${WEBSERVER_FOLDER}/assets/romm && \ ln -sf /romm/resources ${WEBSERVER_FOLDER}/assets/romm/resources COPY ./backend /backend -# Setup init script and config files -COPY ./docker/init_scripts/* / +# s6-overlay supervises every process in the container: ordered startup with +# readiness gates, restart of anything that dies, and a bounded SIGTERM/SIGKILL +# shutdown so a process that refuses to exit cannot strand PID 1. +ARG S6_OVERLAY_VERSION +ARG S6_OVERLAY_NOARCH_SHA256 +ARG S6_OVERLAY_X86_64_SHA256 +ARG S6_OVERLAY_AARCH64_SHA256 +ARG TARGETARCH +RUN apk add --no-cache --virtual .s6-build-deps xz && \ + case "${TARGETARCH}" in \ + amd64) s6_arch="x86_64"; s6_sha="${S6_OVERLAY_X86_64_SHA256}" ;; \ + arm64) s6_arch="aarch64"; s6_sha="${S6_OVERLAY_AARCH64_SHA256}" ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + s6_url="https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}" && \ + wget -qO /tmp/s6-noarch.tar.xz "${s6_url}/s6-overlay-noarch.tar.xz" && \ + echo "${S6_OVERLAY_NOARCH_SHA256} /tmp/s6-noarch.tar.xz" | sha256sum -c - && \ + wget -qO /tmp/s6-arch.tar.xz "${s6_url}/s6-overlay-${s6_arch}.tar.xz" && \ + echo "${s6_sha} /tmp/s6-arch.tar.xz" | sha256sum -c - && \ + tar -C / -Jxpf /tmp/s6-noarch.tar.xz && \ + tar -C / -Jxpf /tmp/s6-arch.tar.xz && \ + rm -f /tmp/s6-noarch.tar.xz /tmp/s6-arch.tar.xz && \ + apk del .s6-build-deps + +# Setup entrypoint, service definitions and config files +COPY ./docker/init_scripts/docker-entrypoint.sh /docker-entrypoint.sh +COPY ./docker/s6-overlay/ /etc/s6-overlay/ COPY ./docker/nginx/js/ /etc/nginx/js/ COPY ./docker/nginx/templates/ /etc/nginx/templates/ COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf @@ -237,6 +266,14 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 ENV PYTHONPATH=/backend +# Exit the container when a startup step fails, instead of leaving it up and +# serving nothing, so the restart policy can retry. +ENV S6_BEHAVIOUR_IF_STAGE2_FAILS=2 +# Upper bounds on shutdown: how long a service gets after SIGTERM before s6 +# stops waiting, and how long anything still alive gets before SIGKILL. +ENV S6_SERVICES_GRACETIME=10000 +ENV S6_KILL_GRACETIME=5000 + # Declare only the parent `/romm`, not its subdirs, so Docker keeps them # on one mount (`st_dev`) and cross-directory `os.link()` hardlinks don't fail. VOLUME ["/romm", "/redis-data"] diff --git a/docker/init_scripts/init b/docker/init_scripts/init deleted file mode 100755 index fb8af6cfff..0000000000 --- a/docker/init_scripts/init +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit # treat errors as fatal -set -o nounset # treat unset variables as an error -set -o pipefail # treat errors in pipes as fatal -shopt -s inherit_errexit # inherit errexit - -LOGLEVEL="${LOGLEVEL:="INFO"}" - -# make it possible to disable the inotify watcher process -ENABLE_RESCAN_ON_FILESYSTEM_CHANGE="${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:="false"}" -ENABLE_SCHEDULED_CLEANUP_ORPHANED_RESOURCES="${ENABLE_SCHEDULED_CLEANUP_ORPHANED_RESOURCES:="false"}" -ENABLE_SCHEDULED_RESCAN="${ENABLE_SCHEDULED_RESCAN:="false"}" -ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA="${ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA:="false"}" -ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB="${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB:="false"}" -ENABLE_SYNC_FOLDER_WATCHER="${ENABLE_SYNC_FOLDER_WATCHER:="false"}" - -# if REDIS_HOST is set, we assume that an external redis is used -REDIS_HOST="${REDIS_HOST:=""}" - -# snapshot policy of the internal valkey, as " " pairs. Saving -# hourly keeps a graceful shutdown lossless while a hard crash can only lose an -# hour of sessions and cached metadata. Set to an empty value to never snapshot. -REDIS_SAVE_POLICY="${REDIS_SAVE_POLICY="3600 1"}" - -# logger colors -RED='\033[0;31m' -LIGHTMAGENTA='\033[0;95m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -BLUE='\033[0;34m' -CYAN='\033[0;36m' -RESET='\033[0;00m' - -print_banner() { - local version - version=$(python3 -c "exec(open('__version__.py').read()); print(__version__)") - info_log " _____ __ __ " - info_log ' | __ \ | \/ |' - info_log ' | |__) |___ _ __ ___ | \ / |' - info_log " | _ // _ \\| '_ \` _ \\| |\\/| |" - info_log ' | | \ \ (_) | | | | | | | | |' - info_log ' |_| \_\___/|_| |_| |_|_| |_|' - info_log "" - info_log "The beautiful, powerful, self-hosted Rom manager and player" - info_log "" - info_log "Version: ${version}" - info_log "" -} - -debug_log() { - # print debug log output if enabled - if [[ ${LOGLEVEL} == "DEBUG" ]]; then - echo -e "${LIGHTMAGENTA}DEBUG: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true - fi -} - -info_log() { - echo -e "${GREEN}INFO: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true -} - -warn_log() { - echo -e "${YELLOW}WARNING: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true -} - -error_log() { - echo -e "${RED}ERROR: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true - exit 1 -} - -# Populate a caller-provided array with the opentelemetry-instrument wrapper -# argv tokens for service "$2", or leave it empty if OTEL is disabled or the -# wrapper binary is missing. Use "${arr[@]}" to exec directly, or -# "${arr[*]@Q}" to embed as a shell-quoted prefix string. -otel_prefix() { - local -n out_arr="$1" - if [[ ${OTEL_SDK_DISABLED:-false} == "true" ]]; then return 0; fi - if ! command -v opentelemetry-instrument >/dev/null 2>&1; then - warn_log "opentelemetry-instrument not found, starting $2 without OpenTelemetry instrumentation" - return 0 - fi - # shellcheck disable=SC2034 # nameref binds out_arr to caller variable - out_arr=(opentelemetry-instrument --service_name "${OTEL_SERVICE_NAME_PREFIX-}$2") -} - -# Commands to run initial startup tasks -run_startup() { - local -a wrap=() - otel_prefix wrap startup - if ! PYTHONPATH="/backend:${PYTHONPATH-}" "${wrap[@]}" python3 /backend/startup.py; then - error_log "Startup script failed, exiting" - fi -} - -wait_for_gunicorn() { - local host="127.0.0.1" - local port="${DEV_PORT:-5000}" - debug_log "Waiting for gunicorn on ${host}:${port}..." - local wait_seconds=${WEB_SERVER_GUNICORN_WAIT_SECONDS:=30} - local retries=$((wait_seconds * 2)) - - # Temporarily disable errexit while polling the TCP port. - set +o errexit - while ((retries > 0)); do - if (echo >/dev/tcp/"${host}"/"${port}") 2>/dev/null; then - set -o errexit - debug_log "Gunicorn is ready and accepting connections" - return 0 - fi - sleep 0.5 - ((retries--)) - done - set -o errexit - - warn_log "Gunicorn did not become ready after waiting ${wait_seconds}s!" -} - -# Runs our main process and creates a corresponding PID file -start_bin_gunicorn() { - # Wire LOGLEVEL into Gunicorn's logging config so it respects the env var. - local gunicorn_level="${LOGLEVEL^^}" - local gunicorn_log_config="/tmp/gunicorn/logging.conf" - # Copy the config to a writable runtime path so this works on read-only root filesystems. - mkdir -p /tmp/gunicorn - cp /etc/gunicorn/logging.conf "${gunicorn_log_config}" - sed -i '/\[logger_gunicorn\]/,/^\[/ s/^level=.*/level='"${gunicorn_level}"'/' "${gunicorn_log_config}" - - # commands to start our main application and store its PID to check for crashes - info_log "Starting backend" - - export PYTHONUNBUFFERED=1 - export PYTHONDONTWRITEBYTECODE=1 - - local -a wrap=() - otel_prefix wrap api - "${wrap[@]}" gunicorn \ - --bind=0.0.0.0:"${DEV_PORT:-5000}" \ - --pid=/tmp/gunicorn.pid \ - --forwarded-allow-ips="*" \ - --worker-class uvicorn_worker.UvicornWorker \ - --workers "${WEB_SERVER_CONCURRENCY:-1}" \ - --timeout "${WEB_SERVER_TIMEOUT:-300}" \ - --keep-alive "${WEB_SERVER_KEEPALIVE:-2}" \ - --max-requests "${WEB_SERVER_MAX_REQUESTS:-1000}" \ - --max-requests-jitter "${WEB_SERVER_MAX_REQUESTS_JITTER:-100}" \ - --worker-connections "${WEB_SERVER_WORKER_CONNECTIONS:-1000}" \ - --error-logfile - \ - --log-config "${gunicorn_log_config}" \ - main:app & -} - -# Commands to start nginx (handling PID creation internally) -start_bin_nginx() { - wait_for_gunicorn - - info_log "Starting nginx" - if [[ ${EUID} -ne 0 ]]; then - nginx - else - # if container runs as root, drop permissions - nginx -g 'user romm;' - fi - - : "${ROMM_BASE_URL:=http://0.0.0.0:8080}" - info_log "🚀 RomM is now available at ${ROMM_BASE_URL}" -} - -# Commands to start valkey-server (handling PID creation internally) -start_bin_valkey-server() { - info_log "Starting internal valkey" - - # Most of the keyspace is bulky, rebuildable metadata (Switch TitleDB, - # LaunchBox), so every snapshot rewrites hundreds of MB of dump.rdb. Valkey's - # stock policy does that every few minutes even on an idle instance, which - # adds up to gigabytes of disk writes per hour. - local -a save_flags=(--save "") - if [[ -n ${REDIS_SAVE_POLICY} ]]; then - # shellcheck disable=SC2206 # each " " token is its own argument - save_flags=(--save ${REDIS_SAVE_POLICY}) - fi - - if [[ -f /usr/local/etc/valkey/valkey.conf ]]; then - if [[ ${LOGLEVEL} == "DEBUG" ]]; then - valkey-server /usr/local/etc/valkey/valkey.conf & - else - valkey-server /usr/local/etc/valkey/valkey.conf >/dev/null 2>&1 & - fi - else - if [[ ${LOGLEVEL} == "DEBUG" ]]; then - valkey-server --dir /redis-data "${save_flags[@]}" & - else - valkey-server --dir /redis-data "${save_flags[@]}" >/dev/null 2>&1 & - fi - fi - - VALKEY_PID=$! - echo "${VALKEY_PID}" >/tmp/valkey-server.pid - - local host="127.0.0.1" - local port="6379" - local max_retries=120 - local retry=0 - - debug_log "Waiting for internal valkey to be ready..." - - # Temporarily disable errexit for this part of the script - set +o errexit - - while ((retry < max_retries)); do - # Attempt to check if valkey TCP port is open - if (echo >/dev/tcp/"${host}"/"${port}") 2>/dev/null; then - debug_log "Internal valkey is ready and accepting connections" - set -o errexit # Re-enable errexit after success - return 0 - fi - - sleep 0.5 - ((retry++)) - done - - error_log "Internal valkey did not become ready after $((max_retries * 500))ms" -} - -# Commands to start RQ scheduler -start_bin_rq_scheduler() { - info_log "Starting RQ scheduler" - - RQ_REDIS_HOST=${REDIS_HOST:-127.0.0.1} \ - RQ_REDIS_PORT=${REDIS_PORT:-6379} \ - RQ_REDIS_USERNAME=${REDIS_USERNAME:-""} \ - RQ_REDIS_PASSWORD=${REDIS_PASSWORD:-""} \ - RQ_REDIS_DB=${REDIS_DB:-0} \ - RQ_REDIS_SSL=${REDIS_SSL:-0} \ - rqscheduler \ - --path /backend \ - --pid /tmp/rq_scheduler.pid & -} - -# Commands to start RQ worker -start_bin_rq_worker() { - info_log "Starting RQ worker" - - # Build Redis URL properly - local redis_url - if [[ -n ${REDIS_PASSWORD-} ]]; then - redis_url="redis${REDIS_SSL:+s}://${REDIS_USERNAME-}:${REDIS_PASSWORD}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" - elif [[ -n ${REDIS_USERNAME-} ]]; then - redis_url="redis${REDIS_SSL:+s}://${REDIS_USERNAME}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" - else - redis_url="redis${REDIS_SSL:+s}://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" - fi - - # Set PYTHONPATH so RQ can find the tasks module. - # Use a worker class that drops the noisy per-sweep "cleaning registries for - # queue" log line. The maintenance interval keeps its default (~10 min) so - # orphaned STARTED jobs and stale workers are still pruned promptly, which the - # watcher's Worker.all() scan dedupe relies on. - PYTHONPATH="/backend:${PYTHONPATH-}" rq worker \ - --path /backend \ - --worker-class handler.rq_worker.RomMWorker \ - --pid /tmp/rq_worker.pid \ - --url "${redis_url}" \ - --results-ttl "${TASK_RESULT_TTL:-86400}" \ - --logging_level "${LOGLEVEL}" \ - high default low & -} - -start_bin_watcher() { - info_log "Starting watcher" - local -a wrap=() - otel_prefix wrap watcher - watchfiles \ - --target-type command \ - "${wrap[*]@Q} python3 watcher.py" \ - /romm/library & - WATCHER_PID=$! - echo "${WATCHER_PID}" >/tmp/watcher.pid -} - -start_bin_sync_watcher() { - info_log "Starting sync folder watcher" - sync_base_path="${ROMM_BASE_PATH:-/romm}/sync" - mkdir -p "${sync_base_path}" - local -a wrap=() - otel_prefix wrap sync_watcher - watchfiles \ - --target-type command \ - "${wrap[*]@Q} python3 sync_watcher.py" \ - "${sync_base_path}" & - SYNC_WATCHER_PID=$! - echo "${SYNC_WATCHER_PID}" >/tmp/sync_watcher.pid -} - -watchdog_process_pid() { - PROCESS=$1 - if [[ -f "/tmp/${PROCESS}.pid" ]]; then - # Check if the pid we last wrote to our state file is actually active - PID=$(cat "/tmp/${PROCESS}.pid") || true - if [[ ! -d "/proc/${PID}" ]]; then - start_bin_"${PROCESS}" - fi - else - # Start process if we dont have a corresponding PID file - start_bin_"${PROCESS}" - fi -} - -stop_process_pid() { - PROCESS=$1 - if [[ -f "/tmp/${PROCESS}.pid" ]]; then - PID=$(cat "/tmp/${PROCESS}.pid") || true - if [[ -d "/proc/${PID}" ]]; then - info_log "Stopping ${PROCESS}" - kill "${PID}" || true - # wait for process exit - while [[ -e "/proc/${PID}" ]]; do sleep 0.1; done - fi - fi -} - -shutdown() { - # shutdown in reverse order - stop_process_pid rq_worker - stop_process_pid rq_scheduler - stop_process_pid sync_watcher - stop_process_pid watcher - stop_process_pid nginx - stop_process_pid gunicorn - stop_process_pid valkey-server -} - -# switch to backend directory -cd /backend || { error_log "/backend directory doesn't seem to exist"; } - -print_banner - -# setup trap handler -exited=0 -trap 'exited=1 && shutdown' SIGINT SIGTERM EXIT - -# clear any leftover PID files -rm /tmp/*.pid -f - -# Disable OpenTelemetry if no OTEL_ prefixed environment variables are set -if ! printenv | grep -q '^OTEL_'; then - info_log "No OpenTelemetry environment variables found, disabling OpenTelemetry SDK" - export OTEL_SDK_DISABLED=true -fi - -# Set ROMM_AUTH_SECRET_KEY if not already set -if [[ -z ${ROMM_AUTH_SECRET_KEY-} ]]; then - ROMM_AUTH_SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))") - warn_log "ROMM_AUTH_SECRET_KEY not set, generating a random secret key!" - export ROMM_AUTH_SECRET_KEY -fi - -# Start Valkey server if REDIS_HOST is not set (which would mean user is using an external Redis/Valkey) -if [[ -z ${REDIS_HOST} ]]; then - watchdog_process_pid valkey-server -else - info_log "REDIS_HOST is set, not starting internal valkey-server" -fi - -# Run needed database migrations once at startup -info_log "Running database migrations" -if alembic upgrade head; then - info_log "Database migrations succeeded" -else - error_log "Failed to run database migrations" -fi - -# Startup process requires database and cache to be already available -run_startup - -# main loop -while ! ((exited)); do - watchdog_process_pid gunicorn - - # only start the scheduler if enabled - if [[ ${ENABLE_SCHEDULED_RESCAN} == "true" || ${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB} == "true" || ${ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA} == "true" || ${ENABLE_SCHEDULED_CLEANUP_ORPHANED_RESOURCES} == "true" ]]; then - watchdog_process_pid rq_scheduler - fi - - watchdog_process_pid rq_worker - - # only start the watcher if enabled - if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE} == "true" ]]; then - watchdog_process_pid watcher - fi - - # only start the sync folder watcher if enabled - if [[ ${ENABLE_SYNC_FOLDER_WATCHER} == "true" ]]; then - watchdog_process_pid sync_watcher - fi - - watchdog_process_pid nginx - - # check for died processes every 5 seconds - sleep 5 -done diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/dependencies.d/romm-gunicorn b/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/dependencies.d/romm-gunicorn new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/type b/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/up b/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/up new file mode 100644 index 0000000000..cc378feace --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-gunicorn-ready/up @@ -0,0 +1 @@ +/etc/s6-overlay/scripts/gunicorn-ready diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn/dependencies.d/romm-startup b/docker/s6-overlay/s6-rc.d/romm-gunicorn/dependencies.d/romm-startup new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn/finish b/docker/s6-overlay/s6-rc.d/romm-gunicorn/finish new file mode 100755 index 0000000000..70a4273a32 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-gunicorn/finish @@ -0,0 +1,2 @@ +#!/command/with-contenv bash +exec /etc/s6-overlay/scripts/sweep-process-group diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn/flag-timeout-killpg b/docker/s6-overlay/s6-rc.d/romm-gunicorn/flag-timeout-killpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn/run b/docker/s6-overlay/s6-rc.d/romm-gunicorn/run new file mode 100755 index 0000000000..a80fea9163 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-gunicorn/run @@ -0,0 +1,44 @@ +#!/command/with-contenv bash + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="gunicorn" +# shellcheck source=SCRIPTDIR/../../scripts/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +record_process_group + +# Wire LOGLEVEL into Gunicorn's logging config so it respects the env var. +gunicorn_level="${LOGLEVEL^^}" +gunicorn_log_config="/tmp/gunicorn/logging.conf" +# Copy the config to a writable runtime path so this works on read-only root filesystems. +mkdir -p /tmp/gunicorn +cp /etc/gunicorn/logging.conf "${gunicorn_log_config}" +sed -i '/\[logger_gunicorn\]/,/^\[/ s/^level=.*/level='"${gunicorn_level}"'/' "${gunicorn_log_config}" + +info_log "Starting backend" + +export PYTHONUNBUFFERED=1 +export PYTHONDONTWRITEBYTECODE=1 + +cd /backend || error_log "/backend directory doesn't seem to exist" + +declare -a wrap=() +otel_prefix wrap api + +exec "${wrap[@]}" gunicorn \ + --bind=0.0.0.0:"${DEV_PORT:-5000}" \ + --forwarded-allow-ips="*" \ + --worker-class uvicorn_worker.UvicornWorker \ + --workers "${WEB_SERVER_CONCURRENCY:-1}" \ + --timeout "${WEB_SERVER_TIMEOUT:-300}" \ + --keep-alive "${WEB_SERVER_KEEPALIVE:-2}" \ + --max-requests "${WEB_SERVER_MAX_REQUESTS:-1000}" \ + --max-requests-jitter "${WEB_SERVER_MAX_REQUESTS_JITTER:-100}" \ + --worker-connections "${WEB_SERVER_WORKER_CONNECTIONS:-1000}" \ + --error-logfile - \ + --log-config "${gunicorn_log_config}" \ + main:app diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn/timeout-kill b/docker/s6-overlay/s6-rc.d/romm-gunicorn/timeout-kill new file mode 100644 index 0000000000..5caff40c4a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-gunicorn/timeout-kill @@ -0,0 +1 @@ +10000 diff --git a/docker/s6-overlay/s6-rc.d/romm-gunicorn/type b/docker/s6-overlay/s6-rc.d/romm-gunicorn/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-gunicorn/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay/s6-rc.d/romm-init/dependencies.d/base b/docker/s6-overlay/s6-rc.d/romm-init/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-init/type b/docker/s6-overlay/s6-rc.d/romm-init/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-init/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/s6-overlay/s6-rc.d/romm-init/up b/docker/s6-overlay/s6-rc.d/romm-init/up new file mode 100644 index 0000000000..4ea826dc9e --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-init/up @@ -0,0 +1 @@ +/etc/s6-overlay/scripts/init diff --git a/docker/s6-overlay/s6-rc.d/romm-migrations/dependencies.d/romm-valkey-ready b/docker/s6-overlay/s6-rc.d/romm-migrations/dependencies.d/romm-valkey-ready new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-migrations/type b/docker/s6-overlay/s6-rc.d/romm-migrations/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-migrations/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/s6-overlay/s6-rc.d/romm-migrations/up b/docker/s6-overlay/s6-rc.d/romm-migrations/up new file mode 100644 index 0000000000..0f5f09dbb3 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-migrations/up @@ -0,0 +1 @@ +/etc/s6-overlay/scripts/migrations diff --git a/docker/s6-overlay/s6-rc.d/romm-nginx/dependencies.d/romm-gunicorn-ready b/docker/s6-overlay/s6-rc.d/romm-nginx/dependencies.d/romm-gunicorn-ready new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-nginx/finish b/docker/s6-overlay/s6-rc.d/romm-nginx/finish new file mode 100755 index 0000000000..70a4273a32 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-nginx/finish @@ -0,0 +1,2 @@ +#!/command/with-contenv bash +exec /etc/s6-overlay/scripts/sweep-process-group diff --git a/docker/s6-overlay/s6-rc.d/romm-nginx/flag-timeout-killpg b/docker/s6-overlay/s6-rc.d/romm-nginx/flag-timeout-killpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-nginx/run b/docker/s6-overlay/s6-rc.d/romm-nginx/run new file mode 100755 index 0000000000..3be5ac47d0 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-nginx/run @@ -0,0 +1,24 @@ +#!/command/with-contenv bash + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="nginx" +# shellcheck source=SCRIPTDIR/../../scripts/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +record_process_group + +info_log "Starting nginx" + +: "${ROMM_BASE_URL:=http://0.0.0.0:8080}" +info_log "🚀 RomM is now available at ${ROMM_BASE_URL}" + +if [[ ${EUID} -ne 0 ]]; then + exec nginx -g 'daemon off;' +fi + +# if container runs as root, drop permissions +exec nginx -g 'daemon off; user romm;' diff --git a/docker/s6-overlay/s6-rc.d/romm-nginx/timeout-kill b/docker/s6-overlay/s6-rc.d/romm-nginx/timeout-kill new file mode 100644 index 0000000000..5caff40c4a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-nginx/timeout-kill @@ -0,0 +1 @@ +10000 diff --git a/docker/s6-overlay/s6-rc.d/romm-nginx/type b/docker/s6-overlay/s6-rc.d/romm-nginx/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-nginx/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/dependencies.d/romm-startup b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/dependencies.d/romm-startup new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/finish b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/finish new file mode 100755 index 0000000000..70a4273a32 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/finish @@ -0,0 +1,2 @@ +#!/command/with-contenv bash +exec /etc/s6-overlay/scripts/sweep-process-group diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/flag-timeout-killpg b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/flag-timeout-killpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/run b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/run new file mode 100755 index 0000000000..d2ab55c3cb --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/run @@ -0,0 +1,27 @@ +#!/command/with-contenv bash + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="rq-scheduler" +# shellcheck source=SCRIPTDIR/../../scripts/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +record_process_group + +# The scheduler always runs. startup.py registers the netplay, upload-tmp and +# zip-cache cleanups unconditionally, and the watcher defers its rescans through +# the scheduler, so gating it on the ENABLE_SCHEDULED_* flags left those jobs +# queued with nothing to execute them. +info_log "Starting RQ scheduler" + +export RQ_REDIS_HOST="${REDIS_HOST:-127.0.0.1}" +export RQ_REDIS_PORT="${REDIS_PORT:-6379}" +export RQ_REDIS_USERNAME="${REDIS_USERNAME:-""}" +export RQ_REDIS_PASSWORD="${REDIS_PASSWORD:-""}" +export RQ_REDIS_DB="${REDIS_DB:-0}" +export RQ_REDIS_SSL="${REDIS_SSL:-0}" + +exec rqscheduler --path /backend diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/timeout-kill b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/timeout-kill new file mode 100644 index 0000000000..5caff40c4a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/timeout-kill @@ -0,0 +1 @@ +10000 diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/type b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-scheduler/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-worker/dependencies.d/romm-startup b/docker/s6-overlay/s6-rc.d/romm-rq-worker/dependencies.d/romm-startup new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-worker/finish b/docker/s6-overlay/s6-rc.d/romm-rq-worker/finish new file mode 100755 index 0000000000..70a4273a32 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-worker/finish @@ -0,0 +1,2 @@ +#!/command/with-contenv bash +exec /etc/s6-overlay/scripts/sweep-process-group diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-worker/flag-timeout-killpg b/docker/s6-overlay/s6-rc.d/romm-rq-worker/flag-timeout-killpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-worker/run b/docker/s6-overlay/s6-rc.d/romm-rq-worker/run new file mode 100755 index 0000000000..79685417b8 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-worker/run @@ -0,0 +1,38 @@ +#!/command/with-contenv bash + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="rq-worker" +# shellcheck source=SCRIPTDIR/../../scripts/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +record_process_group + +info_log "Starting RQ worker" + +# Build Redis URL properly +if [[ -n ${REDIS_PASSWORD-} ]]; then + redis_url="redis${REDIS_SSL:+s}://${REDIS_USERNAME-}:${REDIS_PASSWORD}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" +elif [[ -n ${REDIS_USERNAME-} ]]; then + redis_url="redis${REDIS_SSL:+s}://${REDIS_USERNAME}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" +else + redis_url="redis${REDIS_SSL:+s}://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" +fi + +# Set PYTHONPATH so RQ can find the tasks module. +# Use a worker class that drops the noisy per-sweep "cleaning registries for +# queue" log line. The maintenance interval keeps its default (~10 min) so +# orphaned STARTED jobs and stale workers are still pruned promptly, which the +# watcher's Worker.all() scan dedupe relies on. +export PYTHONPATH="/backend:${PYTHONPATH-}" + +exec rq worker \ + --path /backend \ + --worker-class handler.rq_worker.RomMWorker \ + --url "${redis_url}" \ + --results-ttl "${TASK_RESULT_TTL:-86400}" \ + --logging_level "${LOGLEVEL}" \ + high default low diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-worker/timeout-kill b/docker/s6-overlay/s6-rc.d/romm-rq-worker/timeout-kill new file mode 100644 index 0000000000..5caff40c4a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-worker/timeout-kill @@ -0,0 +1 @@ +10000 diff --git a/docker/s6-overlay/s6-rc.d/romm-rq-worker/type b/docker/s6-overlay/s6-rc.d/romm-rq-worker/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-rq-worker/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay/s6-rc.d/romm-startup/dependencies.d/romm-migrations b/docker/s6-overlay/s6-rc.d/romm-startup/dependencies.d/romm-migrations new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-startup/type b/docker/s6-overlay/s6-rc.d/romm-startup/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-startup/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/s6-overlay/s6-rc.d/romm-startup/up b/docker/s6-overlay/s6-rc.d/romm-startup/up new file mode 100644 index 0000000000..4c054d899a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-startup/up @@ -0,0 +1 @@ +/etc/s6-overlay/scripts/startup diff --git a/docker/s6-overlay/s6-rc.d/romm-sync-watcher/dependencies.d/romm-startup b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/dependencies.d/romm-startup new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-sync-watcher/finish b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/finish new file mode 100755 index 0000000000..70a4273a32 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/finish @@ -0,0 +1,2 @@ +#!/command/with-contenv bash +exec /etc/s6-overlay/scripts/sweep-process-group diff --git a/docker/s6-overlay/s6-rc.d/romm-sync-watcher/flag-timeout-killpg b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/flag-timeout-killpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-sync-watcher/run b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/run new file mode 100755 index 0000000000..bbd27ac60d --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/run @@ -0,0 +1,32 @@ +#!/command/with-contenv bash + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="sync-watcher" +# shellcheck source=SCRIPTDIR/../../scripts/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +record_process_group + +if [[ ${ENABLE_SYNC_FOLDER_WATCHER:-false} != "true" ]]; then + debug_log "Sync folder watcher is disabled, not starting it" + disable_service +fi + +info_log "Starting sync folder watcher" + +sync_base_path="${ROMM_BASE_PATH:-/romm}/sync" +mkdir -p "${sync_base_path}" + +cd /backend || error_log "/backend directory doesn't seem to exist" + +declare -a wrap=() +otel_prefix wrap sync_watcher + +exec watchfiles \ + --target-type command \ + "${wrap[*]@Q} python3 sync_watcher.py" \ + "${sync_base_path}" diff --git a/docker/s6-overlay/s6-rc.d/romm-sync-watcher/timeout-kill b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/timeout-kill new file mode 100644 index 0000000000..5caff40c4a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/timeout-kill @@ -0,0 +1 @@ +10000 diff --git a/docker/s6-overlay/s6-rc.d/romm-sync-watcher/type b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-sync-watcher/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey-ready/dependencies.d/romm-valkey b/docker/s6-overlay/s6-rc.d/romm-valkey-ready/dependencies.d/romm-valkey new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey-ready/type b/docker/s6-overlay/s6-rc.d/romm-valkey-ready/type new file mode 100644 index 0000000000..bdd22a1850 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-valkey-ready/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey-ready/up b/docker/s6-overlay/s6-rc.d/romm-valkey-ready/up new file mode 100644 index 0000000000..0c9a520a51 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-valkey-ready/up @@ -0,0 +1 @@ +/etc/s6-overlay/scripts/valkey-ready diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey/dependencies.d/romm-init b/docker/s6-overlay/s6-rc.d/romm-valkey/dependencies.d/romm-init new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey/finish b/docker/s6-overlay/s6-rc.d/romm-valkey/finish new file mode 100755 index 0000000000..70a4273a32 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-valkey/finish @@ -0,0 +1,2 @@ +#!/command/with-contenv bash +exec /etc/s6-overlay/scripts/sweep-process-group diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey/flag-timeout-killpg b/docker/s6-overlay/s6-rc.d/romm-valkey/flag-timeout-killpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey/run b/docker/s6-overlay/s6-rc.d/romm-valkey/run new file mode 100755 index 0000000000..ab3d622975 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-valkey/run @@ -0,0 +1,48 @@ +#!/command/with-contenv bash + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="valkey" +# shellcheck source=SCRIPTDIR/../../scripts/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +record_process_group + +if [[ -n ${REDIS_HOST:-} ]]; then + info_log "REDIS_HOST is set, not starting internal valkey-server" + disable_service +fi + +info_log "Starting internal valkey" + +# Most of the keyspace is bulky, rebuildable metadata (Switch TitleDB, +# LaunchBox), so every snapshot rewrites hundreds of MB of dump.rdb. Valkey's +# stock policy does that every few minutes even on an idle instance, which +# adds up to gigabytes of disk writes per hour. +# +# Snapshotting hourly keeps a graceful shutdown lossless while a hard crash can +# only lose an hour of sessions and cached metadata. Set REDIS_SAVE_POLICY to an +# empty value to never snapshot. +REDIS_SAVE_POLICY="${REDIS_SAVE_POLICY="3600 1"}" + +declare -a save_flags=(--save "") +if [[ -n ${REDIS_SAVE_POLICY} ]]; then + # shellcheck disable=SC2206 # each " " token is its own argument + save_flags=(--save ${REDIS_SAVE_POLICY}) +fi + +# Valkey aborts its own shutdown when the exit snapshot cannot be written, for +# example on a read-only /redis-data, and would otherwise sit there ignoring +# SIGTERM until s6 escalates to SIGKILL and drops the snapshot entirely. +declare -a args=(--dir /redis-data "${save_flags[@]}" --shutdown-on-sigterm force) +if [[ -f /usr/local/etc/valkey/valkey.conf ]]; then + args=(/usr/local/etc/valkey/valkey.conf --shutdown-on-sigterm force) +fi + +if [[ ${LOGLEVEL^^} == "DEBUG" ]]; then + exec valkey-server "${args[@]}" +fi +exec valkey-server "${args[@]}" >/dev/null 2>&1 diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey/timeout-kill b/docker/s6-overlay/s6-rc.d/romm-valkey/timeout-kill new file mode 100644 index 0000000000..5caff40c4a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-valkey/timeout-kill @@ -0,0 +1 @@ +10000 diff --git a/docker/s6-overlay/s6-rc.d/romm-valkey/type b/docker/s6-overlay/s6-rc.d/romm-valkey/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-valkey/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay/s6-rc.d/romm-watcher/dependencies.d/romm-startup b/docker/s6-overlay/s6-rc.d/romm-watcher/dependencies.d/romm-startup new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-watcher/finish b/docker/s6-overlay/s6-rc.d/romm-watcher/finish new file mode 100755 index 0000000000..70a4273a32 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-watcher/finish @@ -0,0 +1,2 @@ +#!/command/with-contenv bash +exec /etc/s6-overlay/scripts/sweep-process-group diff --git a/docker/s6-overlay/s6-rc.d/romm-watcher/flag-timeout-killpg b/docker/s6-overlay/s6-rc.d/romm-watcher/flag-timeout-killpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/s6-rc.d/romm-watcher/run b/docker/s6-overlay/s6-rc.d/romm-watcher/run new file mode 100755 index 0000000000..fb312bd8a1 --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-watcher/run @@ -0,0 +1,29 @@ +#!/command/with-contenv bash + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="watcher" +# shellcheck source=SCRIPTDIR/../../scripts/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +record_process_group + +if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:-false} != "true" ]]; then + debug_log "Rescan on filesystem change is disabled, not starting the watcher" + disable_service +fi + +info_log "Starting watcher" + +cd /backend || error_log "/backend directory doesn't seem to exist" + +declare -a wrap=() +otel_prefix wrap watcher + +exec watchfiles \ + --target-type command \ + "${wrap[*]@Q} python3 watcher.py" \ + /romm/library diff --git a/docker/s6-overlay/s6-rc.d/romm-watcher/timeout-kill b/docker/s6-overlay/s6-rc.d/romm-watcher/timeout-kill new file mode 100644 index 0000000000..5caff40c4a --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-watcher/timeout-kill @@ -0,0 +1 @@ +10000 diff --git a/docker/s6-overlay/s6-rc.d/romm-watcher/type b/docker/s6-overlay/s6-rc.d/romm-watcher/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/s6-overlay/s6-rc.d/romm-watcher/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay/scripts/gunicorn-ready b/docker/s6-overlay/scripts/gunicorn-ready new file mode 100755 index 0000000000..d3989bb5d8 --- /dev/null +++ b/docker/s6-overlay/scripts/gunicorn-ready @@ -0,0 +1,24 @@ +#!/command/with-contenv bash +# Hold nginx back until the upstream it proxies to is listening. Non-fatal: a +# slow backend should still end up behind a running nginx. + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="gunicorn" +# shellcheck source=SCRIPTDIR/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +port="${DEV_PORT:-5000}" +wait_seconds="${WEB_SERVER_GUNICORN_WAIT_SECONDS:-30}" + +debug_log "Waiting for gunicorn on 127.0.0.1:${port}..." +if wait_for_tcp 127.0.0.1 "${port}" "${wait_seconds}"; then + debug_log "Gunicorn is ready and accepting connections" +else + warn_log "Gunicorn did not become ready after waiting ${wait_seconds}s!" +fi + +exit 0 diff --git a/docker/s6-overlay/scripts/init b/docker/s6-overlay/scripts/init new file mode 100755 index 0000000000..94ea7dc65d --- /dev/null +++ b/docker/s6-overlay/scripts/init @@ -0,0 +1,26 @@ +#!/command/with-contenv bash +# Prepare the environment every other service inherits. + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="init" +# shellcheck source=SCRIPTDIR/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +print_banner + +# Values written here are picked up by every later `with-contenv` service. +S6_ENV="/run/s6/container_environment" + +if ! printenv | grep -q '^OTEL_'; then + info_log "No OpenTelemetry environment variables found, disabling OpenTelemetry SDK" + printf 'true' >"${S6_ENV}/OTEL_SDK_DISABLED" +fi + +if [[ -z ${ROMM_AUTH_SECRET_KEY-} ]]; then + warn_log "ROMM_AUTH_SECRET_KEY not set, generating a random secret key!" + python3 -c "import secrets; print(secrets.token_hex(32), end='')" >"${S6_ENV}/ROMM_AUTH_SECRET_KEY" +fi diff --git a/docker/s6-overlay/scripts/lib.sh b/docker/s6-overlay/scripts/lib.sh new file mode 100644 index 0000000000..862189f6f2 --- /dev/null +++ b/docker/s6-overlay/scripts/lib.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Shared helpers for the s6-rc service scripts. Sourced, never executed. +# Callers set ROMM_LOG_TAG before sourcing to label their log lines. + +LOGLEVEL="${LOGLEVEL:="INFO"}" +ROMM_LOG_TAG="${ROMM_LOG_TAG:="init"}" + +# logger colors +RED='\033[0;31m' +LIGHTMAGENTA='\033[0;95m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +RESET='\033[0;00m' + +_log() { + local level_color=$1 level_text=$2 + shift 2 + echo -e "${level_color}${level_text}${BLUE}[RomM]${LIGHTMAGENTA}[${ROMM_LOG_TAG}]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true +} + +debug_log() { + if [[ ${LOGLEVEL^^} == "DEBUG" ]]; then + _log "${LIGHTMAGENTA}" "DEBUG: " "${@}" + fi +} + +info_log() { + _log "${GREEN}" "INFO: " "${@}" +} + +warn_log() { + _log "${YELLOW}" "WARNING: " "${@}" +} + +# Fatal. A oneshot exiting non-zero fails the s6-rc transition, and +# S6_BEHAVIOUR_IF_STAGE2_FAILS=2 turns that into a container exit, so the +# restart policy can retry instead of leaving a half-started container up. +error_log() { + _log "${RED}" "ERROR: " "${@}" + exit 1 +} + +print_banner() { + local version + version=$(python3 -c "exec(open('/backend/__version__.py').read()); print(__version__)") + info_log " _____ __ __ " + info_log ' | __ \ | \/ |' + info_log ' | |__) |___ _ __ ___ | \ / |' + info_log " | _ // _ \\| '_ \` _ \\| |\\/| |" + info_log ' | | \ \ (_) | | | | | | | | |' + info_log ' |_| \_\___/|_| |_| |_|_| |_|' + info_log "" + info_log "The beautiful, powerful, self-hosted Rom manager and player" + info_log "" + info_log "Version: ${version}" + info_log "" +} + +# Populate a caller-provided array with the opentelemetry-instrument wrapper +# argv tokens for service "$2", or leave it empty if OTEL is disabled or the +# wrapper binary is missing. Use "${arr[@]}" to exec directly, or +# "${arr[*]@Q}" to embed as a shell-quoted prefix string. +otel_prefix() { + local -n out_arr="$1" + if [[ ${OTEL_SDK_DISABLED:-false} == "true" ]]; then return 0; fi + if ! command -v opentelemetry-instrument >/dev/null 2>&1; then + warn_log "opentelemetry-instrument not found, starting $2 without OpenTelemetry instrumentation" + return 0 + fi + # shellcheck disable=SC2034 # nameref binds out_arr to caller variable + out_arr=(opentelemetry-instrument --service_name "${OTEL_SERVICE_NAME_PREFIX-}$2") +} + +# Poll a TCP port until it accepts a connection. Returns non-zero on timeout. +wait_for_tcp() { + local host=$1 port=$2 timeout_seconds=$3 + local retries=$((timeout_seconds * 2)) + while ((retries > 0)); do + if (echo >"/dev/tcp/${host}/${port}") 2>/dev/null; then + return 0 + fi + sleep 0.5 + retries=$((retries - 1)) + done + return 1 +} + +# s6-supervise starts each service in its own session, so the service pid +# doubles as its process group id. Record it while the cwd is still the +# servicedir, so ./finish can sweep the group once the service dies. Call this +# from a run script before it changes directory or execs. +record_process_group() { + mkdir -p /run/romm + echo "$$" >"/run/romm/$(basename "${PWD}").pgid" +} + +# Leave this longrun down for the lifetime of the container. Used by services +# that configuration switches off: s6 has no conditional services, so they +# start, opt out, and tell the supervisor not to bring them back. +disable_service() { + s6-svc -O . + exit 0 +} diff --git a/docker/s6-overlay/scripts/migrations b/docker/s6-overlay/scripts/migrations new file mode 100755 index 0000000000..be71af8b6b --- /dev/null +++ b/docker/s6-overlay/scripts/migrations @@ -0,0 +1,20 @@ +#!/command/with-contenv bash +# Bring the schema up to date once per container start. + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="migrations" +# shellcheck source=SCRIPTDIR/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +cd /backend || error_log "/backend directory doesn't seem to exist" + +info_log "Running database migrations" +if alembic upgrade head; then + info_log "Database migrations succeeded" +else + error_log "Failed to run database migrations" +fi diff --git a/docker/s6-overlay/scripts/startup b/docker/s6-overlay/scripts/startup new file mode 100755 index 0000000000..d19df564b8 --- /dev/null +++ b/docker/s6-overlay/scripts/startup @@ -0,0 +1,20 @@ +#!/command/with-contenv bash +# Startup tasks that need both the database and the cache to be reachable. + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="startup" +# shellcheck source=SCRIPTDIR/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +cd /backend || error_log "/backend directory doesn't seem to exist" + +declare -a wrap=() +otel_prefix wrap startup + +if ! PYTHONPATH="/backend:${PYTHONPATH-}" "${wrap[@]}" python3 /backend/startup.py; then + error_log "Startup script failed, exiting" +fi diff --git a/docker/s6-overlay/scripts/sweep-process-group b/docker/s6-overlay/scripts/sweep-process-group new file mode 100755 index 0000000000..4346db5443 --- /dev/null +++ b/docker/s6-overlay/scripts/sweep-process-group @@ -0,0 +1,21 @@ +#!/command/with-contenv bash +# ./finish for every supervised longrun. +# +# Children of a service that died hard (OOM kill, SIGKILL, segfault) are +# reparented to PID 1 but keep the dead parent's process group, and they keep +# holding its listening sockets. The replacement nginx or gunicorn then fails to +# bind and s6 restarts it forever. Sweep the group before the service comes back. + +set -o nounset + +pgid_file="/run/romm/$(basename "${PWD}").pgid" +[[ -f ${pgid_file} ]] || exit 0 + +pgid=$(cat "${pgid_file}") +rm -f "${pgid_file}" + +# Never widen this to 0 or 1: those signal every process we are allowed to signal. +[[ ${pgid} =~ ^[0-9]+$ ]] && ((pgid > 1)) || exit 0 + +kill -9 -- "-${pgid}" 2>/dev/null || true +exit 0 diff --git a/docker/s6-overlay/scripts/valkey-ready b/docker/s6-overlay/scripts/valkey-ready new file mode 100755 index 0000000000..7768e0d3e7 --- /dev/null +++ b/docker/s6-overlay/scripts/valkey-ready @@ -0,0 +1,21 @@ +#!/command/with-contenv bash +# Gate everything that needs a cache on the internal valkey accepting connections. + +set -o errexit +set -o nounset +set -o pipefail +shopt -s inherit_errexit + +ROMM_LOG_TAG="valkey" +# shellcheck source=SCRIPTDIR/lib.sh +source /etc/s6-overlay/scripts/lib.sh + +if [[ -n ${REDIS_HOST:-} ]]; then + exit 0 +fi + +debug_log "Waiting for internal valkey to be ready..." +if ! wait_for_tcp 127.0.0.1 6379 60; then + error_log "Internal valkey did not become ready after 60s" +fi +debug_log "Internal valkey is ready and accepting connections" diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-gunicorn b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-gunicorn new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-gunicorn-ready b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-gunicorn-ready new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-init b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-init new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-migrations b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-migrations new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-nginx b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-nginx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-rq-scheduler b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-rq-scheduler new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-rq-worker b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-rq-worker new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-startup b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-startup new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-sync-watcher b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-sync-watcher new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-valkey b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-valkey new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-valkey-ready b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-valkey-ready new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/contents.d/romm-watcher b/docker/s6-overlay/user-bundles.d/user/contents.d/romm-watcher new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user/type b/docker/s6-overlay/user-bundles.d/user/type new file mode 100644 index 0000000000..757b422115 --- /dev/null +++ b/docker/s6-overlay/user-bundles.d/user/type @@ -0,0 +1 @@ +bundle diff --git a/docker/s6-overlay/user-bundles.d/user2/contents.d/user b/docker/s6-overlay/user-bundles.d/user2/contents.d/user new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/s6-overlay/user-bundles.d/user2/type b/docker/s6-overlay/user-bundles.d/user2/type new file mode 100644 index 0000000000..757b422115 --- /dev/null +++ b/docker/s6-overlay/user-bundles.d/user2/type @@ -0,0 +1 @@ +bundle diff --git a/docs/BACKEND_ARCHITECTURE.md b/docs/BACKEND_ARCHITECTURE.md index 83b4cedc80..90d3bbce42 100644 --- a/docs/BACKEND_ARCHITECTURE.md +++ b/docs/BACKEND_ARCHITECTURE.md @@ -347,6 +347,35 @@ backend/ ## 4. Application Lifecycle +### Container Process Model + +The image runs several processes, supervised by [s6-overlay](https://github.com/just-containers/s6-overlay) as PID 1. Service definitions live in `docker/s6-overlay/`, and `s6-rc` starts them in dependency order: + +```text +romm-init oneshot banner, OTEL detection, ROMM_AUTH_SECRET_KEY + └── romm-valkey longrun internal cache (skipped when REDIS_HOST is set) + └── romm-valkey-ready oneshot polls 127.0.0.1:6379 + └── romm-migrations oneshot alembic upgrade head + └── romm-startup oneshot startup.py + ├── romm-gunicorn longrun ASGI server + │ └── romm-gunicorn-ready oneshot polls DEV_PORT + │ └── romm-nginx longrun reverse proxy on ROMM_PORT + ├── romm-rq-worker longrun + ├── romm-rq-scheduler longrun (only with a scheduled task enabled) + ├── romm-watcher longrun (ENABLE_RESCAN_ON_FILESYSTEM_CHANGE) + └── romm-sync-watcher longrun (ENABLE_SYNC_FOLDER_WATCHER) +``` + +Behaviour worth knowing: + +- **A failed startup step exits the container** (`S6_BEHAVIOUR_IF_STAGE2_FAILS=2`). An unreachable database at boot is normal when the daemon's restart policy starts containers before their dependencies, so migrations fail, the container exits non-zero, and the restart policy retries. +- **A crashed service is restarted on its own**, without taking the rest of the container down. Its leftover children are swept first (`finish`), so a replacement can rebind the port a hard-killed parent's workers were still holding. +- **Shutdown is bounded.** Each service gets `timeout-kill` (10s) before SIGKILL, escalated to the whole process group (`flag-timeout-killpg`). No process can hold PID 1 open indefinitely. + +Optional services have no conditional mechanism in `s6-rc`: they start, check their environment variable, and take themselves down with `s6-svc -O` when disabled. + +Running with a read-only root filesystem requires `/run` to be mounted **exec**, since s6 executes its generated stage scripts from there. + ### Startup Sequence ```text