Adding realworld test dataset

.github/actions/start-hashtopolis/action.yml

@@ -9,14 +9,41 @@ inputs:
     options:
       - "mysql"
       - "postgres"
+  test_dataset:
+    description: "Test dataset to initialize"
+    required: false
+    default: "synthetic"
+    options:
+      - "synthetic"
+      - "realworld"
 
 runs:
   using: "composite"
   steps:
     - name: Start application containers
+      if: ${{ inputs.test_dataset == 'synthetic' }}
       working-directory: .github
       run: docker compose -f docker-compose.${{ inputs.db_system }}.yml up -d
       shell: bash
+    - name: Build application image for real-world dataset setup
+      if: ${{ inputs.test_dataset == 'realworld' }}
+      working-directory: .github
+      run: docker compose -f docker-compose.${{ inputs.db_system }}.yml build hashtopolis-server-dev
+      shell: bash
+    - name: Start database container for real-world dataset setup
+      if: ${{ inputs.test_dataset == 'realworld' }}
+      working-directory: .github
+      run: docker compose -f docker-compose.${{ inputs.db_system }}.yml up -d hashtopolis-db-dev
+      shell: bash
+    - name: Import real-world test dataset
+      if: ${{ inputs.test_dataset == 'realworld' }}
+      run: bash .github/scripts/import-testdata-dump.sh ${{ inputs.db_system }}
+      shell: bash
+    - name: Start application container after real-world import
+      if: ${{ inputs.test_dataset == 'realworld' }}
+      working-directory: .github
+      run: docker compose -f docker-compose.${{ inputs.db_system }}.yml up -d hashtopolis-server-dev
+      shell: bash
 # should not be needed anymore as it is installed during build
 #    - name: Install composer dependencies packages
 #      run: docker exec hashtopolis-server-dev composer install --working-dir=/var/www/html/

.github/scripts/import-testdata-dump.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  printf 'Usage: %s <mysql|postgres>\n' "$0" >&2
+  exit 2
+fi
+
+db_system=$1
+db_container="hashtopolis-db-dev"
+database="hashtopolis"
+user="hashtopolis"
+password="hashtopolis"
+config_source="ci/testdata/config.json"
+config_target="/usr/local/share/hashtopolis/config/config.json"
+compose_file=".github/docker-compose.${db_system}.yml"
+
+if [[ ! -f "$config_source" ]]; then
+  printf 'Missing testdata config: %s\n' "$config_source" >&2
+  exit 1
+fi
+
+if [[ ! -f "$compose_file" ]]; then
+  printf 'Missing compose file: %s\n' "$compose_file" >&2
+  exit 1
+fi
+
+printf 'Installing real-world test config into application data volume: %s\n' "$config_target"
+docker compose -f "$compose_file" run --rm --no-deps --entrypoint bash -T -u root hashtopolis-server-dev \
+  -c "mkdir -p '$(dirname "$config_target")' && cp '/var/www/html/$config_source' '$config_target' && chown www-data:www-data '$config_target'"
+
+wait_for_mysql() {
+  for _ in {1..90}; do
+    if docker exec "$db_container" mysql -u"$user" -p"$password" "$database" -e 'SELECT 1;' >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+  printf 'MySQL database did not become ready in time.\n' >&2
+  return 1
+}
+
+wait_for_postgres() {
+  for _ in {1..90}; do
+    if docker exec "$db_container" pg_isready -U "$user" -d "$database" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+  printf 'PostgreSQL database did not become ready in time.\n' >&2
+  return 1
+}
+
+case "$db_system" in
+  mysql)
+    dump="ci/testdata/db_dump.mysql"
+    if [[ ! -f "$dump" ]]; then
+      printf 'Missing MySQL test dump: %s\n' "$dump" >&2
+      exit 1
+    fi
+    wait_for_mysql
+    printf 'Importing real-world MySQL test dump: %s\n' "$dump"
+    docker exec -i "$db_container" mysql -u"$user" -p"$password" "$database" < "$dump"
+    ;;
+  postgres)
+    dump="ci/testdata/db_dump.psql"
+    if [[ ! -f "$dump" ]]; then
+      printf 'Missing PostgreSQL test dump: %s\n' "$dump" >&2
+      exit 1
+    fi
+    wait_for_postgres
+    printf 'Resetting PostgreSQL schema before real-world dump import\n'
+    docker exec "$db_container" psql -v ON_ERROR_STOP=1 -U "$user" -d "$database" \
+      -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'
+    printf 'Importing real-world PostgreSQL test dump: %s\n' "$dump"
+    docker exec -i "$db_container" psql -v ON_ERROR_STOP=1 -U "$user" -d "$database" < "$dump"
+    ;;
+  *)
+    printf 'Unsupported db system: %s\n' "$db_system" >&2
+    exit 2
+    ;;
+esac
+
+printf 'Real-world test dataset import completed.\n'

.github/workflows/ci.yml

@@ -37,3 +37,26 @@ jobs:
       - name: Show installed files tree in /var/www/html
         if: ${{ always() }}
         run: docker exec hashtopolis-server-dev find /var/www/html
+
+  realworld:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - db_system: mysql
+          - db_system: postgres
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+      - name: Start Hashtopolis server with real-world test data
+        uses: ./.github/actions/start-hashtopolis
+        with:
+          db_system: ${{ matrix.db_system }}
+          test_dataset: realworld
+      - name: Give Apache permissions on necessary directories
+        run: docker exec -u root hashtopolis-server-dev bash -c "chown -R www-data:www-data /var/www/html/src && chmod -R g+w /var/www/html/src"
+      - name: Test with pytest on real-world test data
+        run: docker exec -w /var/www/html/ci/apiv2 -e HASHTOPOLIS_TEST_DATASET=realworld hashtopolis-server-dev pytest
+      - name: Show docker log files
+        if: ${{ always() }}
+        run: docker logs hashtopolis-server-dev

ci/apiv2/conftest.py

@@ -1,6 +1,33 @@
 import os
 import pytest
 
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--test-dataset',
+        action='store',
+        default=os.getenv('HASHTOPOLIS_TEST_DATASET', 'synthetic'),
+        choices=('synthetic', 'realworld'),
+        help='Dataset profile used by the running Hashtopolis test instance.',
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line('markers', 'realworld: test requires the real-world dump dataset')
+    config.addinivalue_line('markers', 'synthetic_only: test requires the synthetic/default test dataset')
+
+
+def pytest_collection_modifyitems(config, items):
+    dataset = config.getoption('--test-dataset')
+    skip_realworld = pytest.mark.skip(reason='requires --test-dataset=realworld')
+    skip_synthetic = pytest.mark.skip(reason='requires --test-dataset=synthetic')
+
+    for item in items:
+        if 'realworld' in item.keywords and dataset != 'realworld':
+            item.add_marker(skip_realworld)
+        if 'synthetic_only' in item.keywords and dataset != 'synthetic':
+            item.add_marker(skip_synthetic)
+
 if os.getenv('_PYTEST_RAISE', "0") != "0":
 
     @pytest.hookimpl(tryfirst=True)
@@ -9,4 +36,4 @@ def pytest_exception_interact(call):
 
     @pytest.hookimpl(tryfirst=True)
     def pytest_internalerror(excinfo):
-        raise excinfo.value
+        raise excinfo.value

ci/apiv2/test_file.py

@@ -1,3 +1,5 @@
+import pytest
+
 from hashtopolis import File, Helper
 from utils import BaseTest
 
@@ -62,6 +64,7 @@ def test_acl(self):
         model_obj = self.create_test_object()
         self._test_acl_list(model_obj, {'permFileRead': True})
 
+    @pytest.mark.synthetic_only
     def test_helper_rescan_global_files(self):
         model_obj1 = self.create_test_object()
         model_obj2 = self.create_test_object()

ci/apiv2/test_permissions.py

@@ -2,6 +2,7 @@
 import json
 import time
 
+import pytest
 import requests
 from hashtopolis import Agent, Chunk, File, Hash, Hashlist, HashType, HealthCheckAgent, Helper, Task, TaskWrapper, User
 
@@ -1559,6 +1560,7 @@ def test_api_token_rebuild_chunk_cache_helper_allowed_with_config_update_scope(s
         self.assertEqual(response.status_code, 200, response.text)
         self.assertEqual(response.json()['meta']['Rebuild'], 'Success')
 
+    @pytest.mark.synthetic_only
     def test_api_token_rescan_global_files_helper_allowed_with_config_update_scope(self):
         """rescanGlobalFiles succeeds with permConfigUpdate.