chore: better local dev

.github/workflows/autofix.yml

@@ -112,7 +112,6 @@ jobs:
         run: pnpm run type-refresh
         working-directory: webclient
 
-      - run: git pull --rebase
       - uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27 # v1.3.2
         with:
           commit-message: "chore: Update OpenAPI spec and types"

.github/workflows/e2e-tests.yml

@@ -0,0 +1,228 @@
+name: E2E Tests
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, reopened, synchronize]
+  push:
+    branches: [main]
+  workflow_dispatch:
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  api-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+
+    steps:
+      - uses: BRAINSia/free-disk-space@f9f59b0ef974ff4b873aa243ce5278ab644eedc6 # v2.1.2
+        with:
+          tool-cache: false
+          mandb:   true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Checkout code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with: { persist-credentials: false }
+
+      - name: Setup | Pull Docker images
+        run: docker compose pull
+      - name: Setup | Build Docker images
+        run: docker compose -f compose.local.yml build server data
+
+      - name: Setup | pnpm
+        uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
+        with:
+          version: 10.14.0
+      - name: Setup | Node.js
+        uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
+        with:
+          node-version: 24
+          cache: 'pnpm'
+          cache-dependency-path: tests/pnpm-lock.yaml
+
+      - name: Setup | Playwright dependencies
+        working-directory: tests
+        run: pnpm install
+
+      - name: Create .env file for docker-compose
+        run: |
+          cat > .env << EOF
+          POSTGRES_PASSWORD=test_password
+          POSTGRES_USER=postgres
+          POSTGRES_DB=navigatum_test
+          MEILI_MASTER_KEY=test_master_key
+          LOG_LEVEL=debug
+          EOF
+
+      - name: Wait for server to become ready
+        run: |
+          set -euo pipefail
+
+          echo "::group::Starting services…"
+          docker compose -f compose.local.yml up -d
+
+          echo "Current container status:"
+          docker compose -f compose.local.yml ps
+
+          echo "Waiting for API to become healthy (max 120s)…"
+
+          timeout 120 bash -c 'until curl -f http://localhost:3003/api/search?q=MI; do sleep 2; done'
+          if ! timeout 120 bash -c '
+            until curl -fs http://localhost:3003/api/locations/mi >/dev/null; do
+              echo "Still waiting for API…"
+              sleep 2
+            done
+          '; then
+            echo "::endgroup::"
+            echo "::error::Timed out waiting for API to become healthy"
+
+            echo "::group::Container status:"
+            docker compose -f compose.local.yml ps
+            echo "::endgroup::"
+
+            echo "::group::Recent container logs:"
+            docker compose -f compose.local.yml logs --tail=100
+            echo "::endgroup::"
+
+            exit 1
+          fi
+
+          echo "::group::API is responding"
+          curl -v -f http://localhost:3003/api/status
+          echo "::endgroup::"
+
+      - name: Run API tests
+        working-directory: tests
+        run: pnpm test:api
+        env:
+          SKIP_WEBSERVER: true
+          BASE_URL: http://localhost:3003
+
+      - name: Upload test results (JUnit)
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: api-test-results
+          path: tests/test-results/junit.xml
+          retention-days: 30
+
+      - name: Server logs on failure
+        if: failure()
+        run: docker compose -f compose.local.yml logs server
+      - name: MeiliSearch logs on failure
+        if: failure()
+        run: docker compose -f compose.local.yml logs meilisearch
+      - name: Database logs on failure
+        if: failure()
+        run: docker compose -f compose.local.yml logs db
+
+  ui-tests-desktop:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium]
+        subsystem: [main, search, navigation, details, calendar-about]
+
+    steps:
+      - uses: BRAINSia/free-disk-space@f9f59b0ef974ff4b873aa243ce5278ab644eedc6 # v2.1.2
+        with:
+          tool-cache: false
+          mandb:   true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+      - name: Checkout code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with: { persist-credentials: false }
+
+      - name: Setup | Pull Docker images
+        run: docker compose pull
+      - name: Setup | Build Docker images
+        run: docker compose -f compose.local.yml build
+
+      - name: Setup | Set up pnpm
+        uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
+        with:
+          version: 10.14.0
+      - name: Setup | Set up Node.js
+        uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
+        with:
+          node-version: 24
+          cache: 'pnpm'
+          cache-dependency-path: tests/pnpm-lock.yaml
+
+      - name: Setup | Install Playwright dependencies
+        working-directory: tests
+        run: pnpm install
+      - name: Install Browser
+        working-directory: tests
+        run: pnpm exec playwright install --with-deps ${{ matrix.browser }}
+
+      - name: Setup | Create .env file for docker-compose
+        run: |
+          cat > .env << EOF
+          POSTGRES_PASSWORD=test_password
+          POSTGRES_USER=postgres
+          POSTGRES_DB=navigatum_test
+          MEILI_MASTER_KEY=test_master_key
+          LOG_LEVEL=debug
+          EOF
+
+      - name: Setup | Start Docker services
+        run: |
+          docker compose -f compose.local.yml up -d
+          echo "Waiting for services to be healthy..."
+          timeout 120 bash -c 'until curl -f http://localhost:3003/api/locations/mi; do sleep 2; done'
+          timeout 120 bash -c 'until curl -f http://localhost:3003/api/search?q=MI; do sleep 2; done'
+          timeout 120 bash -c 'until curl -f http://localhost:3000/; do sleep 2; done'
+
+      - name: Run UI tests (${{ matrix.subsystem }})
+        working-directory: tests
+        run: pnpm test specs/${{ matrix.subsystem }}.ui.spec.ts --project=ui-tests-${{ matrix.browser }}
+        env:
+          BASE_URL: http://localhost:3000
+          SKIP_WEBSERVER: true
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: ui-playwright-report-${{ matrix.browser }}-${{ matrix.subsystem }}
+          path: tests/playwright-report/
+          retention-days: 30
+
+      - name: Upload test results (JUnit)
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: ui-test-results-${{ matrix.browser }}-${{ matrix.subsystem }}
+          path: tests/test-results/junit.xml
+          retention-days: 30
+
+      - name: Server logs on failure
+        if: failure()
+        run: docker compose -f compose.local.yml logs server
+      - name: MeiliSearch logs on failure
+        if: failure()
+        run: docker compose -f compose.local.yml logs meilisearch
+      - name: Database logs on failure
+        if: failure()
+        run: docker compose -f compose.local.yml logs db

.github/workflows/security.yaml

@@ -45,6 +45,7 @@ jobs:
           # RUSTSEC-2025-0004 = ssl::select_next_proto use after free => not used
           # RUSTSEC-2025-0006 = Hickory DNS failure to verify self-signed RRSIG for DNSKEYs => not used
           # RUSTSEC-2025-0134 = rustls-pemfile is unmaintained, but only used in development dependencies
+          working-directory: server
   codeql:
     name: CodeQL (${{ matrix.language }})
     runs-on: ubuntu-latest

compose.local.yml

@@ -6,16 +6,10 @@ services:
       - "3000:3000"
     environment:
       TZ: Europe/Berlin
-      NUXT_PUBLIC_CDN_URL: http://data:3002
+      NUXT_PUBLIC_CDN_URL: http://server:3003
       NUXT_PUBLIC_API_URL: http://server:3003
-  # cdn
-  data:
-    restart: unless-stopped
-    build: ./data
-    environment:
-      TZ: Europe/Berlin
-    ports:
-      - "3002:3002"
+      # calendar takes for ever and needs permissions
+      NUXT_PUBLIC_CALENDAR_URL: https://nav.tum.de
   server:
     restart: unless-stopped
     build:
@@ -32,7 +26,7 @@ services:
       LOG_LEVEL: ${LOG_LEVEL-debug}
       MIELI_URL: http://meilisearch:7700
       MEILI_MASTER_KEY: ${MEILI_MASTER_KEY}
-      CDN_URL: http://data:3002/cdn
+      CDN_URL: http://server:3003/cdn
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_DB: ${POSTGRES_DB}

compose.yml

@@ -16,30 +16,15 @@ services:
       TZ: Europe/Berlin
       NUXT_PUBLIC_CDN_URL: https://nav.tum.de
       NUXT_PUBLIC_API_URL: https://nav.tum.de
-  # cdn
-  data:
-    image: ghcr.io/tum-dev/navigatum-data:main
-    restart: unless-stopped
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.navigatum-data.entrypoints=webs"
-      - "traefik.http.routers.navigatum-data.tls.certresolver=leacme"
-      - "traefik.http.routers.navigatum-data.rule=Host(`nav.tum.de`) && PathPrefix(`/cdn/`)"
-      - "traefik.http.services.navigatum-data.loadbalancer.server.port=3002"
-    networks:
-      - traefik_traefik
-    environment:
-      TZ: Europe/Berlin
-    expose:
-      - "3002"
+      NUXT_PUBLIC_CALENDAR_URL: https://nav.tum.de
   server:
     image: ghcr.io/tum-dev/navigatum-server:main
     restart: unless-stopped
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.navigatum-server.entrypoints=webs"
       - "traefik.http.routers.navigatum-server.tls.certresolver=leacme"
-      - "traefik.http.routers.navigatum-server.rule=Host(`nav.tum.de`) && PathPrefix(`/api`)"
+      - "traefik.http.routers.navigatum-server.rule=Host(`nav.tum.de`) && (PathPrefix(`/api`) || PathPrefix(`/cdn`))"
       - "traefik.http.services.navigatum-server.loadbalancer.server.port=3003"
     networks:
       - traefik_traefik
@@ -52,7 +37,7 @@ services:
       LOG_LEVEL: ${LOG_LEVEL-info}
       MIELI_URL: http://meilisearch:7700
       MEILI_MASTER_KEY: ${MEILI_MASTER_KEY}
-      CDN_URL: http://data:3002/cdn
+      CDN_URL: http://server:3003/cdn
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_DB: ${POSTGRES_DB}

data/Dockerfile

@@ -30,14 +30,11 @@ RUN cp external/results/public_transport.parquet output/public_transport.parquet
     && cp -r output/maps/overlays output/maps/overlay
 # last renaming is to remain backwards compatible
 
-# compress data (only using gzip, because brotli on ngnix is a royal pain)
+# Compress data (only using gzip, because brotli is complex)
 RUN gzip --force --keep --recursive output/
 
-FROM nginx:1.29 AS production-stage
-RUN mkdir /cdn
-COPY --from=build-stage /app/output /cdn
-COPY nginx.conf /etc/nginx/nginx.conf
+# Create final output directory structure for extraction
+RUN mkdir -p /cdn && cp -r output/* /cdn/
 
-EXPOSE 3002
-HEALTHCHECK CMD curl --fail http://localhost:3002/cdn/health || exit 1
-CMD ["nginx", "-g", "daemon off;"]
+# This image is meant to be used as a data source in multi-stage builds
+# No runtime server is needed - the server image will copy files from /cdn

data/README.md

@@ -8,6 +8,8 @@ This folder contains:
 
 The code to retrieve external data as well as externally retrieved data is located under `external`.
 
+**Note:** The data Docker image is used as a build-only artifact. The compiled data files are copied into the server image during its build process. The data container does not run as a service in production.
+
 > [!WARNING]
 > A lot of this code is more a work-in-progress than finished.
 > Especially features such as POIs, custom maps or other data types such as events are drafted but not yet fully implemented.
@@ -89,11 +91,12 @@ Deployment related there are also these files:
 
 ```bash
 data
-├── Dockerfile       # Main dockerfile, in the deployment this is sometimes called the cdn
-├── ngnix.conf       # nginx configuration file used by above Dockerfile
-└── requirements.txt # python dependency's
+├── Dockerfile       # Dockerfile for compiling data (used as source in server multi-stage build)
+└── requirements.txt # Python dependencies
 ```
 
+The compiled data is automatically included in the server Docker image during build and served at `/cdn`.
+
 ### How the data looks like
 
 ```json

data/output/openapi.yaml

@@ -385,13 +385,6 @@ paths:
               schema:
                 type: string
               example: Invalid ID
-        "404":
-          description: '**Not found.** Make sure that requested item exists'
-          content:
-            text/plain:
-              schema:
-                type: string
-              example: Not found
         "500":
           description: '**Internal server error**'
           content: