diff --git a/.github/actions/get-job-data/action.yml b/.github/actions/get-job-data/action.yml new file mode 100644 index 0000000000000..a0b15c0dd8030 --- /dev/null +++ b/.github/actions/get-job-data/action.yml @@ -0,0 +1,93 @@ +name: 'Get Job Data from GitHub Actions' +description: 'Fetches the GitHub Actions job data from the GitHub API' +inputs: + job-name: + description: 'The name of the job to find' + required: true + github-token: + description: 'GitHub token for API authentication' + required: true + repository: + description: 'Repository in owner/repo format' + required: true + run-id: + description: 'GitHub Actions run ID' + required: true +outputs: + job_html_url: + description: 'The HTML URL of the job' + value: ${{ steps.get_url.outputs.job_html_url }} +runs: + using: 'composite' + steps: + - name: Fetch job URL from GitHub API + id: get_url + shell: bash + run: | + # Fetch the numeric job ID from GitHub API + CURL_ERROR_FILE=$(mktemp) + NETRC_FILE=$(mktemp) + + # Write GitHub API credentials to a temporary netrc file to avoid + # passing the token directly on the curl command line. + printf '%s\n' \ + 'machine api.github.com' \ + ' login x-access-token' \ + " password ${{ inputs.github-token }}" \ + > "$NETRC_FILE" + chmod 600 "$NETRC_FILE" + + # Ensure temporary file cleanup on exit + cleanup() { + if [ -n "$CURL_ERROR_FILE" ] && [ -f "$CURL_ERROR_FILE" ]; then + rm -f "$CURL_ERROR_FILE" + fi + if [ -n "$NETRC_FILE" ] && [ -f "$NETRC_FILE" ]; then + rm -f "$NETRC_FILE" + fi + } + trap cleanup EXIT + + API_RESPONSE=$(curl -sS -w "\n%{http_code}" \ + --netrc-file "$NETRC_FILE" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ inputs.repository }}/actions/runs/${{ inputs.run-id }}/jobs" 2>"$CURL_ERROR_FILE") + + CURL_EXIT_CODE=$? + if [ $CURL_EXIT_CODE -ne 0 ]; then + echo "❌ ERROR: curl request to GitHub API failed with exit code $CURL_EXIT_CODE" + if [ -s "$CURL_ERROR_FILE" ]; then + echo "curl error output:" + cat "$CURL_ERROR_FILE" + fi + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + HTTP_CODE=$(echo "$API_RESPONSE" | tail -n1) + RESPONSE_BODY=$(echo "$API_RESPONSE" | sed '$d') + + if [ "$HTTP_CODE" != "200" ]; then + echo "⚠️ WARNING: GitHub API request failed with $HTTP_CODE" + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + EXPECTED_JOB_NAME="${{ inputs.job-name }}" + JOB_URL=$(echo "$RESPONSE_BODY" | jq -r \ + --arg job_name "$EXPECTED_JOB_NAME" \ + '.jobs[] | select(.name == $job_name) | .html_url') + + if [ -z "$JOB_URL" ] || [ "$JOB_URL" = "null" ]; then + echo "⚠️ WARNING: Failed to extract job URL from response for job name '$EXPECTED_JOB_NAME'." + echo "Possible causes:" + echo " - The job name does not match exactly (including spaces and case)." + echo " - The job has not started yet at the time this action ran." + echo " - The GitHub API response format was unexpected." + echo "Please verify that the job has started before this action runs and double-check the exact job name in the GitHub Actions UI." + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "job_html_url=$JOB_URL" >> "$GITHUB_OUTPUT" + echo "Job URL: $JOB_URL" diff --git a/.github/scripts/detect-app-cache.sh b/.github/scripts/detect-app-cache.sh new file mode 100755 index 0000000000000..0803d27156a63 --- /dev/null +++ b/.github/scripts/detect-app-cache.sh @@ -0,0 +1,377 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2025 STRATO AG +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Script to detect which apps need building vs. can be restored from cache +# Supports multiple cache sources: GitHub Actions cache and JFrog Artifactory +# Outputs JSON arrays for apps to build and apps to restore + +set -e # Exit on error +set -u # Exit on undefined variable +set -o pipefail # Exit if any command in pipeline fails + +# Required environment variables +: "${GH_TOKEN:?GH_TOKEN not set}" +: "${CACHE_VERSION:?CACHE_VERSION not set}" +: "${FORCE_REBUILD:?FORCE_REBUILD not set}" +: "${ARTIFACTORY_REPOSITORY_SNAPSHOT:?ARTIFACTORY_REPOSITORY_SNAPSHOT not set}" + +# Optional variables +APPS_TO_REBUILD="${APPS_TO_REBUILD:-}" +JF_URL="${JF_URL:-}" +JF_USER="${JF_USER:-}" +JF_ACCESS_TOKEN="${JF_ACCESS_TOKEN:-}" + +# Input: MATRIX (JSON array of app configurations) +# Input: GITHUB_REF (current GitHub ref) +# Input: GITHUB_STEP_SUMMARY (path to step summary file) + +# Outputs to $GITHUB_OUTPUT: +# - apps_to_build: JSON array of apps that need building +# - apps_to_restore: JSON array of apps that can be restored from cache +# - apps_sha_map: JSON object mapping app names to their SHAs +# - has_apps_to_build: boolean flag +# - has_apps_to_restore: boolean flag + +echo "Collecting app SHAs and checking cache status..." +echo "Cache version: $CACHE_VERSION" +echo "Force rebuild mode: $FORCE_REBUILD" +if [ -n "$APPS_TO_REBUILD" ]; then + echo "Apps to rebuild: $APPS_TO_REBUILD" +fi +echo "" + +# Setup JFrog CLI if credentials are available +JFROG_AVAILABLE="false" +echo "=== JFrog Setup ===" +echo "JF_URL present: $([ -n "$JF_URL" ] && echo 'YES' || echo 'NO')" +echo "JF_USER present: $([ -n "$JF_USER" ] && echo 'YES' || echo 'NO')" +echo "JF_ACCESS_TOKEN present: $([ -n "$JF_ACCESS_TOKEN" ] && echo 'YES' || echo 'NO')" + +if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "✓ All JFrog credentials available" + if ! command -v jf >/dev/null 2>&1; then + echo "⚠ JFrog CLI (jf) not in PATH; skipping JFrog cache checks" + echo " (jf should be installed by jfrog/setup-jfrog-cli before this script runs)" + else + echo "JFrog CLI version: $(jf --version)" + + # Configure JFrog (skip if already configured — avoids "Server ID already exists" when + # a previous step in the same job has already run jf config add with the same server ID) + if jf config show jfrog-server >/dev/null 2>&1; then + echo "JFrog server 'jfrog-server' already configured, reusing" + else + echo "Configuring JFrog server: $JF_URL" + jf config add jfrog-server --url="$JF_URL" --user="$JF_USER" --access-token="$JF_ACCESS_TOKEN" --interactive=false + fi + + # Test connection with verbose output + echo "Testing JFrog connection..." + if jf rt ping; then + JFROG_AVAILABLE="true" + echo "✓ JFrog connection successful" + echo "Repository: $ARTIFACTORY_REPOSITORY_SNAPSHOT" + else + echo "⚠ JFrog ping failed, will fall back to GitHub cache" + fi + fi +else + echo "⚠ JFrog credentials not available, using GitHub cache only" + [ -z "$JF_URL" ] && echo " - Missing: JF_URL" + [ -z "$JF_USER" ] && echo " - Missing: JF_USER" + [ -z "$JF_ACCESS_TOKEN" ] && echo " - Missing: JF_ACCESS_TOKEN" +fi +echo "JFROG_AVAILABLE=$JFROG_AVAILABLE" +echo "===================" +echo "" + +# Get the matrix from input (passed as argument) +MATRIX="$1" + +# Build JSON arrays for apps that need building/restoring +APPS_TO_BUILD="[]" +APPS_TO_RESTORE="[]" +APPS_CHECKED=0 +APPS_CACHED=0 +APPS_IN_JFROG=0 +APPS_TO_BUILD_COUNT=0 +APPS_TO_RESTORE_COUNT=0 +APPS_SHA_MAP="{}" +echo "" + +echo "### 📦 Cache Status Report for ($GITHUB_REF)" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" +echo "**Cache Version:** \`$CACHE_VERSION\`" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" +if [ "$FORCE_REBUILD" == "true" ]; then + echo "**🔄 FORCE REBUILD MODE ENABLED** - All caches bypassed" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +if [ -n "$APPS_TO_REBUILD" ]; then + echo "**🔨 Specific apps to rebuild:** \`$APPS_TO_REBUILD\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +if [ "$JFROG_AVAILABLE" == "true" ]; then + echo "**🎯 JFrog Artifact Cache**: Enabled for all branches" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +echo "| App | SHA | Cache Key | Status |" >> "$GITHUB_STEP_SUMMARY" +echo "|-----|-----|-----------|--------|" >> "$GITHUB_STEP_SUMMARY" + +# Convert comma-separated apps list to array for easier checking +if [ -n "$APPS_TO_REBUILD" ]; then + IFS=',' read -ra REBUILD_APPS_ARRAY <<< "$APPS_TO_REBUILD" + # Trim whitespace from each app name using xargs for readability + for i in "${!REBUILD_APPS_ARRAY[@]}"; do + REBUILD_APPS_ARRAY[$i]=$(echo "${REBUILD_APPS_ARRAY[$i]}" | xargs) + done +else + REBUILD_APPS_ARRAY=() +fi + +# Iterate through each app in the matrix +while IFS= read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_PATH=$(echo "$app_json" | jq -r '.path') + + APPS_CHECKED=$((APPS_CHECKED + 1)) + + # Get current submodule SHA + if [ -d "$APP_PATH" ]; then + CURRENT_SHA=$(git -C "$APP_PATH" rev-parse HEAD 2>/dev/null || echo "") + else + echo "⊘ $APP_NAME - directory not found at '$APP_PATH' (submodule not initialised?), will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Directory not found — will build |" >> "$GITHUB_STEP_SUMMARY" + UNKNOWN_SUFFIX="unknown" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$UNKNOWN_SUFFIX" \ + --arg archive_name "${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + if [ -z "$CURRENT_SHA" ]; then + echo "⊘ $APP_NAME - '$APP_PATH' exists but is not a git repo, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Not a git repo — will build |" >> "$GITHUB_STEP_SUMMARY" + UNKNOWN_SUFFIX="unknown" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$UNKNOWN_SUFFIX" \ + --arg archive_name "${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Add SHA to the map for all apps (regardless of cache status) + APPS_SHA_MAP=$(echo "$APPS_SHA_MAP" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '.[$app] = $sha') + + # Cache key for this app — use 8-char short SHAs throughout. + SHORT_SHA="${CURRENT_SHA:0:8}" + CACHE_KEY="${CACHE_VERSION}-app-build-${APP_NAME}-${SHORT_SHA}" + ARCHIVE_SUFFIX="${SHORT_SHA}" + + echo -n " Checking $APP_NAME (SHA: $SHORT_SHA)... " + + # If force rebuild is enabled, skip cache check and rebuild everything + if [ "$FORCE_REBUILD" == "true" ]; then + echo "🔄 force rebuild" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔄 Force rebuild |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Check if this specific app should be rebuilt (ignore cache) + REBUILD_THIS_APP=false + for rebuild_app in "${REBUILD_APPS_ARRAY[@]}"; do + if [ "$APP_NAME" == "$rebuild_app" ]; then + REBUILD_THIS_APP=true + break + fi + done + + if [ "$REBUILD_THIS_APP" == "true" ]; then + echo "🔨 rebuild requested" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Rebuild requested |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Check JFrog first before GitHub cache (available for all branches) + if [ "$JFROG_AVAILABLE" == "true" ]; then + JFROG_PATH="${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" + FOUND_IN_JFROG="false" + + echo "" + echo " 🔍 Checking JFrog for $APP_NAME..." + echo " Path: $JFROG_PATH" + echo " SHA: $SHORT_SHA" + + # Check if artifact exists in JFrog with verbose output + echo " Running: jf rt s \"$JFROG_PATH\"" + if SEARCH_OUTPUT=$(jf rt s "$JFROG_PATH" 2>&1); then + echo " Search exit code: 0" + echo " Search output:" + echo "$SEARCH_OUTPUT" | sed 's/^/ /' + + if echo "$SEARCH_OUTPUT" | grep -q "$JFROG_PATH"; then + echo " ✓ Artifact found in JFrog!" + ARCHIVE_NAME="${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" + echo "✓ in JFrog" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$JFROG_PATH\` | 📦 In JFrog |" >> "$GITHUB_STEP_SUMMARY" + APPS_IN_JFROG=$((APPS_IN_JFROG + 1)) + APPS_TO_RESTORE_COUNT=$((APPS_TO_RESTORE_COUNT + 1)) + # Add to restore list with JFrog source + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --argjson app "$app_json" --arg sha "$CURRENT_SHA" --arg jfrog_path "$JFROG_PATH" --arg archive_name "$ARCHIVE_NAME" --arg source "jfrog" '. + [($app + {sha: $sha, jfrog_path: $jfrog_path, archive_name: $archive_name, source: $source})]') + FOUND_IN_JFROG="true" + else + echo " ✗ Artifact not found in search results" + fi + else + echo " Search exit code: non-zero" + echo " ✗ Search failed with error:" + echo "$SEARCH_OUTPUT" | sed 's/^/ /' + fi + + if [ "$FOUND_IN_JFROG" == "true" ]; then + continue + fi + + echo "⚡ needs build (JFrog miss)" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$JFROG_PATH\` | 🔨 Needs build (not in JFrog) |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Check if cache exists using GitHub CLI + # Include --ref to access caches from the current ref (branch, PR, etc.) + if ! CACHE_LIST=$(gh cache list --ref "$GITHUB_REF" --key "$CACHE_KEY" --json key --jq ".[].key" 2>&1); then + echo "⚠️ Warning: Failed to query cache for $APP_NAME: $CACHE_LIST" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ⚠️ Cache check failed - will build |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + if echo "$CACHE_LIST" | grep -q "^${CACHE_KEY}$"; then + APPS_CACHED=$((APPS_CACHED + 1)) + APPS_TO_RESTORE_COUNT=$((APPS_TO_RESTORE_COUNT + 1)) + echo "✓ cached" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ✅ Cached |" >> "$GITHUB_STEP_SUMMARY" + # Add to restore list with GitHub cache source + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --argjson app "$app_json" --arg sha "$CURRENT_SHA" --arg cache_key "$CACHE_KEY" --arg source "github-cache" '. + [($app + {sha: $sha, cache_key: $cache_key, source: $source})]') + else + echo "⚡ needs build" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Needs build |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + fi + +done < <(echo "$MATRIX" | jq -c '.[]') + +echo "" >> "$GITHUB_STEP_SUMMARY" +echo "**Summary:**" >> "$GITHUB_STEP_SUMMARY" +echo "- Total apps checked: $APPS_CHECKED" >> "$GITHUB_STEP_SUMMARY" +echo "- 📦 Apps in JFrog: $APPS_IN_JFROG" >> "$GITHUB_STEP_SUMMARY" +echo "- ✅ Apps with cached builds: $APPS_CACHED" >> "$GITHUB_STEP_SUMMARY" +echo "- 🔨 Apps needing build: $APPS_TO_BUILD_COUNT" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" + +TOTAL_AVAILABLE=$((APPS_IN_JFROG + APPS_CACHED)) +if [ $TOTAL_AVAILABLE -gt 0 ] && [ $APPS_CHECKED -gt 0 ]; then + CACHE_HIT_PERCENT=$((TOTAL_AVAILABLE * 100 / APPS_CHECKED)) + echo "**Cache hit rate: ${CACHE_HIT_PERCENT}%** 🎯" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi + +echo "" +echo "Summary:" +echo " Total apps: $APPS_CHECKED" +echo " In JFrog: $APPS_IN_JFROG" +echo " Cached: $APPS_CACHED" +echo " To build: $APPS_TO_BUILD_COUNT" + +# Validate no duplicate apps in build and restore lists +BUILD_APPS=$(echo "$APPS_TO_BUILD" | jq -r '.[].name' | sort) +RESTORE_APPS=$(echo "$APPS_TO_RESTORE" | jq -r '.[].name' | sort) +DUPLICATE_APPS=$(comm -12 <(echo "$BUILD_APPS") <(echo "$RESTORE_APPS")) + +if [ -n "$DUPLICATE_APPS" ]; then + echo "ERROR: Apps appear in both build and restore lists:" + echo "$DUPLICATE_APPS" + exit 1 +fi + +# Validate that we built valid JSON +if ! echo "$APPS_TO_BUILD" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_build" + echo "Content: $APPS_TO_BUILD" + exit 1 +fi + +if ! echo "$APPS_TO_RESTORE" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_restore" + echo "Content: $APPS_TO_RESTORE" + exit 1 +fi + +# Output app list with SHAs for the build job to use +# Use proper multiline output format for GitHub Actions +echo "apps_to_build<> "$GITHUB_OUTPUT" +echo "$APPS_TO_BUILD" >> "$GITHUB_OUTPUT" +echo "APPS_TO_BUILD_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output the unified list of apps to restore (from either GitHub cache or JFrog) +echo "apps_to_restore<> "$GITHUB_OUTPUT" +echo "$APPS_TO_RESTORE" >> "$GITHUB_OUTPUT" +echo "APPS_TO_RESTORE_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output the SHA map for all apps +echo "apps_sha_map<> "$GITHUB_OUTPUT" +echo "$APPS_SHA_MAP" >> "$GITHUB_OUTPUT" +echo "APPS_SHA_MAP_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output flags for conditional job execution +if [ $APPS_TO_BUILD_COUNT -gt 0 ]; then + echo "has_apps_to_build=true" >> "$GITHUB_OUTPUT" +else + echo "has_apps_to_build=false" >> "$GITHUB_OUTPUT" +fi + +if [ $APPS_TO_RESTORE_COUNT -gt 0 ]; then + echo "has_apps_to_restore=true" >> "$GITHUB_OUTPUT" +else + echo "has_apps_to_restore=false" >> "$GITHUB_OUTPUT" +fi + +echo "" +if [ $APPS_TO_BUILD_COUNT -eq 0 ]; then + echo "🎉 All apps are cached! No builds needed." +else + echo "✓ Will build $APPS_TO_BUILD_COUNT app(s)" +fi diff --git a/.github/workflows/hidrive-next-build.yml b/.github/workflows/hidrive-next-build.yml index ed09dd411f362..4a072992352f6 100644 --- a/.github/workflows/hidrive-next-build.yml +++ b/.github/workflows/hidrive-next-build.yml @@ -4,9 +4,10 @@ name: HiDrive Next Build # SPDX-FileCopyrightText: 2024 STRATO AG # SPDX-License-Identifier: AGPL-3.0-or-later -# The HiDrive Next source is packaged as a container image. -# This is a workaround because releases can not be created without tags -# and we want to be able to create snapshots from branches. +# Cached parallel matrix build pipeline: +# Stage 1: prepare-matrix — compute app build matrix from IONOS/Makefile and probe per-app caches +# Stage 2: build-apps — parallel matrix per app, per-app SHA caching in JFrog + GitHub Actions cache +# Final: hidrive-next-build — restore all per-app artifacts and build Nextcloud core on: pull_request: @@ -31,19 +32,440 @@ on: branches: - ionos-dev - ionos-stable + - 'rc/**' + - '*/dev/*' + workflow_dispatch: + inputs: + force_rebuild: + description: 'Force rebuild all apps and dependencies (bypass ALL caches)' + required: false + type: boolean + default: false + cache_version_suffix: + description: 'Optional cache version suffix (e.g., "test", "debug") - creates separate cache namespace' + required: false + type: string + default: '' + apps_to_rebuild: + description: 'Comma-separated list of specific apps to rebuild (e.g., "simplesettings,viewer")' + required: false + type: string + default: '' + +# Concurrency group is intentionally shared between push and pull_request runs +# of the same source branch, so a `*/dev/*` branch with an open PR does not +# produce two parallel runs. Protected branches (ionos-dev/stable/rc/*) use a +# unique-per-run-id key so consecutive pushes never cancel each other. +concurrency: + group: >- + ${{ github.workflow }}-${{ + ( + contains(fromJson('["refs/heads/ionos-dev","refs/heads/ionos-stable"]'), github.ref) || + startsWith(github.ref, 'refs/heads/rc/') + ) && github.run_id || + (github.head_ref || github.ref_name) + }} + cancel-in-progress: true env: TARGET_PACKAGE_NAME: hidrive-next.zip REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} ARTIFACTORY_REPOSITORY_SNAPSHOT: ionos-productivity-hdnext-snapshot + # Cache version - increment to invalidate all caches when build tooling changes + # Format: v. (e.g., v1.0, v1.1, v2.0) + CACHE_VERSION: v1.2 permissions: contents: read jobs: + prepare-matrix: + runs-on: ubuntu-latest + outputs: + apps_to_build: ${{ steps.detect.outputs.apps_to_build }} + apps_to_restore: ${{ steps.detect.outputs.apps_to_restore }} + apps_matrix: ${{ steps.set_matrix.outputs.matrix }} + apps_sha_map: ${{ steps.detect.outputs.apps_sha_map }} + has_apps_to_build: ${{ steps.detect.outputs.has_apps_to_build }} + has_apps_to_restore: ${{ steps.detect.outputs.has_apps_to_restore }} + effective_cache_version: ${{ steps.compute_cache_version.outputs.effective_cache_version }} + + permissions: + contents: read + actions: read + + steps: + - name: Compute effective cache version + id: compute_cache_version + run: | + EFFECTIVE_VERSION="${{ env.CACHE_VERSION }}${{ github.event.inputs.cache_version_suffix && format('-{0}', github.event.inputs.cache_version_suffix) || '' }}" + echo "effective_cache_version=$EFFECTIVE_VERSION" >> "$GITHUB_OUTPUT" + echo "Effective cache version: $EFFECTIVE_VERSION" + + - name: Checkout repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + submodules: true + fetch-depth: 1 + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y make jq + + - name: Check configuration + run: | + echo "### 🔧 Remote Trigger Configuration" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**DISABLE_REMOTE_TRIGGER value:** \`${{ vars.DISABLE_REMOTE_TRIGGER }}\`" >> $GITHUB_STEP_SUMMARY + echo "**ENABLE_REMOTE_TRIGGER_USER_DEV value:** \`${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}\`" >> $GITHUB_STEP_SUMMARY + if [ "${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}" != "true" ]; then + echo " - 💡 To enable the GitLab trigger for \`*/dev/*\` branches, set repository variable \`ENABLE_REMOTE_TRIGGER_USER_DEV\` to \`true\` at [Settings → Variables → Actions](https://github.com/${{ github.repository }}/settings/variables/actions)." >> $GITHUB_STEP_SUMMARY + fi + echo "**Event type:** \`${{ github.event_name }}\`" >> $GITHUB_STEP_SUMMARY + echo "**Branch:** \`${{ github.ref_name }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "🔧 Remote Trigger Configuration" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "DISABLE_REMOTE_TRIGGER = '${{ vars.DISABLE_REMOTE_TRIGGER }}'" + echo "ENABLE_REMOTE_TRIGGER_USER_DEV = '${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}'" + if [ "${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}" != "true" ]; then + echo " 💡 To enable the GitLab trigger for '*/dev/*' branches," + echo " set repository variable ENABLE_REMOTE_TRIGGER_USER_DEV to 'true' at:" + echo " https://github.com/${{ github.repository }}/settings/variables/actions" + fi + echo "Event type = '${{ github.event_name }}'" + echo "Branch = '${{ github.ref_name }}'" + echo "" + + if [ "${{ vars.DISABLE_REMOTE_TRIGGER }}" == "true" ]; then + echo "⚠️ DISABLE_REMOTE_TRIGGER='true' — remote trigger is force-disabled" + echo " The 'trigger-remote-dev-workflow' job will be SKIPPED" + echo "**Status:** ⚠️ Remote trigger is **force-disabled** via \`DISABLE_REMOTE_TRIGGER='true'\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The \`trigger-remote-dev-workflow\` job will be skipped." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "To enable, delete the variable or set it to a value other than 'true' at:" >> $GITHUB_STEP_SUMMARY + echo "https://github.com/${{ github.repository }}/settings/variables/actions" >> $GITHUB_STEP_SUMMARY + else + echo "ℹ️ DISABLE_REMOTE_TRIGGER not set — checking trigger conditions..." + echo "**Status:** \`DISABLE_REMOTE_TRIGGER\` not set — actual outcome determined by conditions below." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + WILL_TRIGGER=true + echo "**Trigger Conditions Check:**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ github.event_name }}" != "push" ]; then + echo "- ❌ Event must be 'push' (current: \`${{ github.event_name }}\`)" >> $GITHUB_STEP_SUMMARY + echo " ❌ Event type is '${{ github.event_name }}' (must be 'push')" + WILL_TRIGGER=false + else + echo "- ✅ Event is 'push'" >> $GITHUB_STEP_SUMMARY + echo " ✅ Event type is 'push'" + fi + + VALID_BRANCH_PATTERN='^(ionos-dev|ionos-stable)$|^rc/.*$|^[^/]+/dev/.*$' + USER_DEV_PATTERN='^[^/]+/dev/.*$' + if [[ ! "${{ github.ref_name }}" =~ $VALID_BRANCH_PATTERN ]]; then + echo "- ❌ Branch must be 'ionos-dev', 'ionos-stable', 'rc/*' or '*/dev/*' (current: \`${{ github.ref_name }}\`)" >> $GITHUB_STEP_SUMMARY + echo " ❌ Branch is '${{ github.ref_name }}' (must be 'ionos-dev', 'ionos-stable', 'rc/*' or '*/dev/*')" + WILL_TRIGGER=false + else + echo "- ✅ Branch is '\`${{ github.ref_name }}\`'" >> $GITHUB_STEP_SUMMARY + echo " ✅ Branch is '${{ github.ref_name }}'" + fi + + if [[ "${{ github.ref_name }}" =~ $USER_DEV_PATTERN ]]; then + if [ "${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}" == "true" ]; then + echo "- ✅ User-dev branch opt-in (\`ENABLE_REMOTE_TRIGGER_USER_DEV='true'\`)" >> $GITHUB_STEP_SUMMARY + echo " ✅ ENABLE_REMOTE_TRIGGER_USER_DEV='true' — '*/dev/*' trigger is opted in" + else + echo "- ❌ User-dev branch requires \`ENABLE_REMOTE_TRIGGER_USER_DEV='true'\` (current: \`${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}\`)" >> $GITHUB_STEP_SUMMARY + echo " ❌ '*/dev/*' branch requires ENABLE_REMOTE_TRIGGER_USER_DEV='true' (current: '${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}')" + WILL_TRIGGER=false + fi + fi + + echo "- ℹ️ All dependent jobs must succeed (checked at job runtime)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "$WILL_TRIGGER" = "true" ]; then + echo "**Expected:** The \`trigger-remote-dev-workflow\` job **WILL RUN** (if all dependent jobs succeed)." >> $GITHUB_STEP_SUMMARY + echo "🎯 Expected: trigger-remote-dev-workflow job WILL RUN (if all dependent jobs succeed)" + else + echo "**Expected:** The \`trigger-remote-dev-workflow\` job **WILL BE SKIPPED** due to unmet conditions above." >> $GITHUB_STEP_SUMMARY + echo "⏭️ Expected: trigger-remote-dev-workflow job WILL BE SKIPPED" + fi + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + - name: List caches before restore + run: gh cache list + env: + GH_TOKEN: ${{ github.token }} + + - name: Check JFrog credentials + id: jfrog-available + if: github.event.inputs.force_rebuild != 'true' + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + run: | + if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "available=true" >> $GITHUB_OUTPUT + else + echo "available=false" >> $GITHUB_OUTPUT + echo "⚠ Full JFrog credential set not available — skipping JFrog checks" + fi + + - name: Setup JFrog CLI + if: steps.jfrog-available.outputs.available == 'true' + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Generate apps matrix from Makefile + id: set_matrix + run: | + echo "Generating apps matrix from Makefile..." + matrix_output=$(make -f IONOS/Makefile generate_apps_matrix_json 2>&1) + if echo "$matrix_output" | grep -q '^\[i\]'; then + matrix=$(echo "$matrix_output" | grep -v '^\[i\]') + else + matrix="$matrix_output" + fi + if ! echo "$matrix" | jq empty 2>/dev/null; then + echo "Error: Generated matrix is not valid JSON" + echo "Output: $matrix_output" + exit 1 + fi + echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT + echo "Matrix generated with $(echo "$matrix" | jq 'length') apps" + + - name: Collect apps SHA and check cache status + id: detect + env: + GH_TOKEN: ${{ github.token }} + CACHE_VERSION: ${{ steps.compute_cache_version.outputs.effective_cache_version }} + FORCE_REBUILD: ${{ github.event.inputs.force_rebuild || 'false' }} + APPS_TO_REBUILD: ${{ github.event.inputs.apps_to_rebuild || '' }} + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + ARTIFACTORY_REPOSITORY_SNAPSHOT: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }} + GITHUB_REF: ${{ github.ref }} + run: | + bash .github/scripts/detect-app-cache.sh '${{ steps.set_matrix.outputs.matrix }}' + + build-apps: + runs-on: ubuntu-latest + needs: prepare-matrix + if: | + always() && + needs.prepare-matrix.result == 'success' && + needs.prepare-matrix.outputs.has_apps_to_build == 'true' + + permissions: + contents: read + actions: write + + name: build-apps + strategy: + max-parallel: 7 + matrix: + app_info: ${{ fromJson(needs.prepare-matrix.outputs.apps_to_build) }} + + steps: + - name: Get app configuration from full matrix + id: app-config + run: | + FULL_MATRIX='${{ needs.prepare-matrix.outputs.apps_matrix }}' + APP_NAME='${{ matrix.app_info.name }}' + + APP_CONFIG=$(echo "$FULL_MATRIX" | jq -c --arg name "$APP_NAME" '.[] | select(.name == $name)') + if [ -z "$APP_CONFIG" ]; then + echo "ERROR: Could not find configuration for $APP_NAME in matrix" + exit 1 + fi + + echo "path=$(echo "$APP_CONFIG" | jq -r '.path')" >> $GITHUB_OUTPUT + echo "has-npm=$(echo "$APP_CONFIG" | jq -r '.has_npm')" >> $GITHUB_OUTPUT + echo "has-composer=$(echo "$APP_CONFIG" | jq -r '.has_composer')" >> $GITHUB_OUTPUT + echo "npm-lock-path=$(echo "$APP_CONFIG" | jq -r '.npm_lock_path')" >> $GITHUB_OUTPUT + echo "makefile-target=$(echo "$APP_CONFIG" | jq -r '.makefile_target')" >> $GITHUB_OUTPUT + echo "Building $APP_NAME (SHA: ${{ matrix.app_info.sha }})" + + - name: Checkout repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + submodules: true + fetch-depth: 1 + + - name: Check JFrog credentials + id: jfrog-creds + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + run: | + if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "available=true" >> $GITHUB_OUTPUT + else + echo "available=false" >> $GITHUB_OUTPUT + echo "⚠ Full JFrog credential set not available — using GitHub cache path where possible" + fi + + - name: Setup JFrog CLI + if: steps.jfrog-creds.outputs.available == 'true' + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Set up node + if: steps.app-config.outputs.has-npm == 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: "package.json" + cache: 'npm' + cache-dependency-path: ${{ steps.app-config.outputs.npm-lock-path }} + + - name: Setup PHP + if: steps.app-config.outputs.has-composer == 'true' + uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 + with: + php-version: '8.3' + tools: composer:v2 + extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache + env: + runner: ubuntu-latest + + - name: Cache Composer dependencies for ${{ matrix.app_info.name }} + if: steps.app-config.outputs.has-composer == 'true' && github.event.inputs.force_rebuild != 'true' + uses: actions/cache@v4 + with: + path: ${{ steps.app-config.outputs.path }}/vendor + key: ${{ runner.os }}-composer-${{ matrix.app_info.name }}-${{ hashFiles(format('{0}/composer.lock', steps.app-config.outputs.path)) }} + restore-keys: | + ${{ runner.os }}-composer-${{ matrix.app_info.name }}- + + - name: Build ${{ matrix.app_info.name }} + env: + CYPRESS_INSTALL_BINARY: 0 + PUPPETEER_SKIP_DOWNLOAD: true + run: make -f IONOS/Makefile ${{ steps.app-config.outputs.makefile-target }} + + - name: Report build completion + if: success() + run: | + echo "### ✅ Built ${{ matrix.app_info.name }}" >> $GITHUB_STEP_SUMMARY + echo "- **SHA:** \`${{ matrix.app_info.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Path:** ${{ steps.app-config.outputs.path }}" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** Success" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Compute app cache key + id: app-cache-key + run: | + APP_SHA="${{ matrix.app_info.sha }}" + APP_NAME="${{ matrix.app_info.name }}" + EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}" + SHORT_SHA="${APP_SHA:0:8}" + + echo "cache_key=${EFFECTIVE_VERSION}-app-build-${APP_NAME}-${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "archive_name=${APP_NAME}-${SHORT_SHA}.tar.gz" >> $GITHUB_OUTPUT + + - name: Get Job data + id: get_job_data + continue-on-error: true + uses: ./.github/actions/get-job-data + with: + job-name: 'build-apps (${{ matrix.app_info.name }}, ${{ matrix.app_info.sha }})' + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + + - name: Upload ${{ matrix.app_info.name }} to JFrog + if: steps.jfrog-creds.outputs.available == 'true' + run: | + APP_NAME="${{ matrix.app_info.name }}" + APP_SHA="${{ matrix.app_info.sha }}" + APP_PATH="${{ steps.app-config.outputs.path }}" + EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}" + ARCHIVE_NAME="${{ steps.app-cache-key.outputs.archive_name }}" + JFROG_PATH="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/apps/${EFFECTIVE_VERSION}/${APP_NAME}/${ARCHIVE_NAME}" + + echo "Packaging $APP_NAME..." + tar -czf "$ARCHIVE_NAME" \ + --exclude="node_modules" \ + --exclude=".git" \ + --exclude="*.log" \ + -C "$(dirname "$APP_PATH")" \ + "$(basename "$APP_PATH")" + + echo "Archive size: $(ls -lh "$ARCHIVE_NAME" | awk '{print $5}')" + echo "Uploading to: $JFROG_PATH" + + JFROG_PROPS_LIST=() + JFROG_PROPS_LIST+=("app.name=${APP_NAME}") + JFROG_PROPS_LIST+=("app.sha=${APP_SHA}") + JFROG_PROPS_LIST+=("vcs.branch=${{ github.ref_name }}") + JFROG_PROPS_LIST+=("vcs.revision=${{ github.sha }}") + JOB_URL="${{ steps.get_job_data.outputs.job_html_url }}" + if [ -n "$JOB_URL" ]; then + JFROG_PROPS_LIST+=("job.html_url=${JOB_URL}") + fi + JFROG_PROPS=$(IFS=';'; printf '%s' "${JFROG_PROPS_LIST[*]}") + + if jf rt upload "$ARCHIVE_NAME" "$JFROG_PATH" --target-props "$JFROG_PROPS"; then + echo "✅ Uploaded $APP_NAME to JFrog" + echo "Verifying upload..." + if jf rt s "$JFROG_PATH" 2>/dev/null | grep -q "$JFROG_PATH"; then + echo "✓ Upload verified — artifact is accessible at $JFROG_PATH" + else + echo "⚠ Upload reported success but verification search did not find the artifact" + fi + else + echo "❌ JFrog upload failed" + exit 1 + fi + rm -f "$ARCHIVE_NAME" + + - name: Save build to GitHub Actions cache + uses: actions/cache/save@v4 + with: + path: ${{ steps.app-config.outputs.path }} + key: ${{ steps.app-cache-key.outputs.cache_key }} + + - name: Upload ${{ matrix.app_info.name }} build artifacts + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: app-build-${{ matrix.app_info.name }} + path: | + ${{ steps.app-config.outputs.path }} + !${{ steps.app-config.outputs.path }}/node_modules + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 + hidrive-next-build: runs-on: ubuntu-latest + needs: [prepare-matrix, build-apps] + if: | + always() && + needs.prepare-matrix.result == 'success' && + (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') permissions: contents: read @@ -54,14 +476,143 @@ jobs: name: hidrive-next-build steps: - name: Checkout server - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: submodules: true + - name: Check JFrog credentials + id: jfrog-creds + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + run: | + if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "available=true" >> $GITHUB_OUTPUT + else + echo "available=false" >> $GITHUB_OUTPUT + echo "⚠ Full JFrog credential set not available — artifact restore is not possible" + fi + + - name: Assert JFrog credentials are available (required for artifact restore) + if: steps.jfrog-creds.outputs.available != 'true' + run: | + echo "❌ JFrog credentials are required for hidrive-next-build artifact restore." + echo " Set JF_ARTIFACTORY_URL, JF_ARTIFACTORY_USER, and JF_ACCESS_TOKEN secrets." + exit 1 + + - name: Setup JFrog CLI + if: steps.jfrog-creds.outputs.available == 'true' + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Ping JFrog server + if: steps.jfrog-creds.outputs.available == 'true' + run: jf rt ping + + - name: Restore all apps from JFrog and GitHub cache + if: steps.jfrog-creds.outputs.available == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}" + FULL_MATRIX='${{ needs.prepare-matrix.outputs.apps_matrix }}' + + # Restore cached apps (from detect-app-cache.sh output) + if [ "${{ needs.prepare-matrix.outputs.has_apps_to_restore }}" == "true" ]; then + echo "Restoring cached apps..." + APPS_TO_RESTORE='${{ needs.prepare-matrix.outputs.apps_to_restore }}' + while read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_PATH=$(echo "$app_json" | jq -r '.path') + SOURCE=$(echo "$app_json" | jq -r '.source') + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "Restoring: $APP_NAME (source: $SOURCE)" + + if [ "$SOURCE" == "jfrog" ]; then + JFROG_PATH=$(echo "$app_json" | jq -r '.jfrog_path') + ARCHIVE_NAME=$(echo "$app_json" | jq -r '.archive_name // empty') + if [ -z "$ARCHIVE_NAME" ]; then + ARCHIVE_NAME="$(basename "$JFROG_PATH")" + fi + echo "Downloading from JFrog: $JFROG_PATH" + jf rt download "$JFROG_PATH" "$ARCHIVE_NAME" --flat=true + mkdir -p "$(dirname "$APP_PATH")" + tar -xzf "$ARCHIVE_NAME" -C "$(dirname "$APP_PATH")" + rm -f "$ARCHIVE_NAME" + if [ ! -d "$APP_PATH" ]; then + echo "❌ Restore validation failed: $APP_PATH not found" + exit 1 + fi + # appinfo/info.xml lives in standard Nextcloud apps but not in themes/ + case "$APP_PATH" in + apps-*) + if [ ! -f "$APP_PATH/appinfo/info.xml" ]; then + echo "❌ Restore validation failed: $APP_PATH/appinfo/info.xml missing" + exit 1 + fi + ;; + esac + echo "✅ Restored $APP_NAME from JFrog" + + elif [ "$SOURCE" == "github-cache" ]; then + CACHE_KEY=$(echo "$app_json" | jq -r '.cache_key') + echo "❌ Cannot restore $APP_NAME from GitHub cache within a shell step." + echo " Cache key: $CACHE_KEY" + echo " GitHub Actions cache requires 'actions/cache/restore@v4' as a dedicated workflow step." + exit 1 + fi + done < <(echo "$APPS_TO_RESTORE" | jq -c '.[]') + fi + + # Restore newly built apps from JFrog (apps_to_build) + if [ "${{ needs.prepare-matrix.outputs.has_apps_to_build }}" == "true" ]; then + echo "Restoring newly built apps from JFrog..." + APPS_TO_BUILD='${{ needs.prepare-matrix.outputs.apps_to_build }}' + while read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + ARCHIVE_NAME=$(echo "$app_json" | jq -r '.archive_name') + JFROG_PATH=$(echo "$app_json" | jq -r '.jfrog_path') + APP_PATH=$(echo "$FULL_MATRIX" | jq -r --arg name "$APP_NAME" '.[] | select(.name == $name) | .path') + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "Restoring newly built: $APP_NAME" + echo "Downloading from JFrog: $JFROG_PATH" + + jf rt download "$JFROG_PATH" "$ARCHIVE_NAME" --flat=true + mkdir -p "$(dirname "$APP_PATH")" + tar -xzf "$ARCHIVE_NAME" -C "$(dirname "$APP_PATH")" + rm -f "$ARCHIVE_NAME" + + if [ ! -d "$APP_PATH" ]; then + echo "❌ Restore validation failed: $APP_PATH not found" + exit 1 + fi + # appinfo/info.xml lives in standard Nextcloud apps but not in themes/ + case "$APP_PATH" in + apps-*) + if [ ! -f "$APP_PATH/appinfo/info.xml" ]; then + echo "❌ Restore validation failed: $APP_PATH/appinfo/info.xml missing" + exit 1 + fi + ;; + esac + echo "✅ Restored $APP_NAME from JFrog" + done < <(echo "$APPS_TO_BUILD" | jq -c '.[]') + fi + + echo "✅ All apps restored" + - name: Set up node with version from package.json's engines - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version-file: "package.json" + cache: ${{ github.event.inputs.force_rebuild != 'true' && 'npm' || '' }} - name: Install Dependencies run: sudo apt-get update && sudo apt-get install -y make zip unzip @@ -72,6 +623,7 @@ jobs: - name: Setup PHP with PECL extension uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 with: + php-version: '8.3' tools: composer:v2 extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache env: @@ -80,41 +632,8 @@ jobs: - name: Print PHP install run: php -i && php -m - - name: Build Nextcloud - run: make -f IONOS/Makefile build_nextcloud - - - name: Install dependencies & build simplesettings app - env: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_DOWNLOAD: true - run: make -f IONOS/Makefile build_dep_simplesettings_app - - - name: Install dependencies & build viewer app - env: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_DOWNLOAD: true - run: make -f IONOS/Makefile build_dep_viewer_app - - - name: Install dependencies & build richdocuments app - run: make -f IONOS/Makefile build_richdocuments_app - - - name: Install dependencies & build user_oidc app - env: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_DOWNLOAD: true - run: make -f IONOS/Makefile build_dep_user_oidc_app - - - name: Install dependencies for external apps nc_ionos_processes - run: make -f IONOS/Makefile build_dep_nc_ionos_processes_app - - - name: Build Custom CSS - run: make -f IONOS/Makefile build_dep_theming_app - - - name: Install dependencies & build IONOS theme custom elements - env: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_DOWNLOAD: true - run: make -f IONOS/Makefile build_dep_ionos_theme + - name: Build Nextcloud (core only — apps restored from cache) + run: make -f IONOS/Makefile build_nextcloud_only - name: Add config partials run: make -f IONOS/Makefile add_config_partials @@ -125,7 +644,6 @@ jobs: run: | echo "${{ github.run_number }}" > .buildnumber echo "✅ Build number injected: ${{ github.run_number }}" - echo "📄 File created: .buildnumber" cat .buildnumber - name: Zip dependencies @@ -161,11 +679,19 @@ jobs: upload-to-artifactory: runs-on: self-hosted - # Upload the artifact to the Artifactory repository on PR *OR* on "ionos-dev|ionos-stable" branch push defined in the on:push:branches - if: github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' + # Upload the artifact to the Artifactory repository on PR *OR* on + # "ionos-dev|ionos-stable|rc/*|*/dev/*" branch push *OR* on manual workflow_dispatch + if: | + always() && + (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || + github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' || + startsWith(github.ref_name, 'rc/') || contains(github.ref_name, '/dev/')) && + needs.prepare-matrix.result == 'success' && + (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') && + needs.hidrive-next-build.result == 'success' name: Push to artifactory - needs: hidrive-next-build + needs: [prepare-matrix, build-apps, hidrive-next-build] outputs: ARTIFACTORY_LAST_BUILD_PATH: ${{ steps.artifactory_upload.outputs.ARTIFACTORY_LAST_BUILD_PATH }} @@ -174,8 +700,39 @@ jobs: BUILD_NAME: "hidrive_next-snapshot" steps: + - name: Check prerequisites + run: | + echo "Checking if required secrets are set..." + error_count=0 + + if [ -z "${{ secrets.JF_ARTIFACTORY_URL }}" ]; then + echo "::error::JF_ARTIFACTORY_URL secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ secrets.JF_ARTIFACTORY_USER }}" ]; then + echo "::error::JF_ARTIFACTORY_USER secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ secrets.JF_ACCESS_TOKEN }}" ]; then + echo "::error::JF_ACCESS_TOKEN secret is not set" + error_count=$((error_count + 1)) + fi + + if [ $error_count -ne 0 ]; then + echo "::error::Required secrets are not set. Aborting." + exit 1 + fi + + # Checkout is required to access the local composite action at ./.github/actions/get-job-data + - name: Checkout repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + fetch-depth: 1 + - name: Download artifact zip - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: hidrive_next_build_artifact @@ -191,34 +748,99 @@ jobs: # Ping the server jf rt ping + - name: Get Job data + id: get_job_data + continue-on-error: true + uses: ./.github/actions/get-job-data + with: + job-name: 'Push to artifactory' + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + - name: Upload build to artifactory id: artifactory_upload run: | - # PR builds are stored in a separate directory as "dev/pr/hidrive-next-pr-.zip" - # Push to "ionos-dev" branch is stored as "dev/hidrive-next-.zip" + # Artifactory Build Storage Structure: + # | Branch/Event | Stage Prefix | Artifact Path | + # |------------------|------------------|------------------------------------------------------------------------------| + # | Pull Request | pr | pr/hidrive-next-pr-.zip | + # | ionos-dev | dev | dev/hidrive-next-//hidrive-next-.zip | + # | ionos-stable | stable | stable/hidrive-next-//hidrive-next-.zip | + # | rc/* | | rc//hidrive-next-//hidrive-next-.zip | + # | */dev/* | devs/ | devs//hidrive-next-//hidrive-next-.zip | ARTIFACTORY_STAGE_PREFIX="dev" - # set ARTIFACTORY_STAGE_PREFIX=stable on ionos-stable branch - if [ ${{ github.ref_name }} == "ionos-stable" ]; then + if [ -n "${{ github.event.pull_request.number }}" ]; then + ARTIFACTORY_STAGE_PREFIX="pr" + elif [ "${{ github.ref_name }}" == "ionos-stable" ]; then ARTIFACTORY_STAGE_PREFIX="stable" + elif [[ "${{ github.ref_name }}" =~ ^rc/.*$ ]]; then + ARTIFACTORY_STAGE_PREFIX="${{ github.ref_name }}" + elif [[ "${{ github.ref_name }}" =~ ^.*/dev/.*$ ]]; then + BRANCH_PREFIX=$(echo "${{ github.ref_name }}" | sed 's|/.*||' | sed 's/[^A-Za-z0-9._-]/-/g') + ARTIFACTORY_STAGE_PREFIX="devs/${BRANCH_PREFIX}" fi export PATH_TO_DIRECTORY="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/${ARTIFACTORY_STAGE_PREFIX}" - PATH_TO_FILE="pr/hidrive-next-pr-${{ github.event.pull_request.number }}.zip" - if [ -z "${{ github.event.pull_request.number }}" ]; then - PATH_TO_FILE="hidrive-next-${{ needs.hidrive-next-build.outputs.NC_VERSION }}.zip" + if [ -n "${{ github.event.pull_request.number }}" ]; then + # PR uploads keep the flat layout (one slot per PR number, top-level pr/) + PATH_TO_FILE="hidrive-next-pr-${{ github.event.pull_request.number }}.zip" + else + # branch uploads nest under // to preserve every build + SHORT_SHA="${{ github.sha }}" + SHORT_SHA="${SHORT_SHA:0:7}" + NC_VERSION="${{ needs.hidrive-next-build.outputs.NC_VERSION }}" + PATH_TO_FILE="hidrive-next-${NC_VERSION}/${SHORT_SHA}/hidrive-next-${NC_VERSION}.zip" fi export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" - # Promote current build to the "latest" dev build - jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ - --build-name "${{ env.BUILD_NAME }}" \ - --build-number ${{ github.run_number }} \ - --target-props "hdnext.nc_version=${{ needs.hidrive-next-build.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ - $PATH_TO_LATEST_ARTIFACT + JFROG_PROPS_LIST=() + JFROG_PROPS_LIST+=("build.nc_version=${{ needs.hidrive-next-build.outputs.NC_VERSION }}") + JFROG_PROPS_LIST+=("vcs.branch=${{ github.ref }}") + JFROG_PROPS_LIST+=("vcs.revision=${{ github.sha }}") + JOB_URL="${{ steps.get_job_data.outputs.job_html_url }}" + if [ -n "$JOB_URL" ]; then + JFROG_PROPS_LIST+=("job.html_url=${JOB_URL}") + fi + JFROG_PROPS=$(IFS=';'; printf '%s' "${JFROG_PROPS_LIST[*]}") + + # Upload with retry logic (3 attempts with exponential backoff: 10s, 20s) + MAX_ATTEMPTS=3 + ATTEMPT=1 + UPLOAD_SUCCESS=false + DELAY_SEC=10 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Upload attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ + --build-name "${{ env.BUILD_NAME }}" \ + --build-number ${{ github.run_number }} \ + --target-props "$JFROG_PROPS" \ + $PATH_TO_LATEST_ARTIFACT; then + UPLOAD_SUCCESS=true + echo "✅ Upload successful on attempt $ATTEMPT" + break + else + echo "⚠️ Upload attempt $ATTEMPT failed" + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting $DELAY_SEC seconds before retry..." + sleep $DELAY_SEC + DELAY_SEC=$((DELAY_SEC * 2)) + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$UPLOAD_SUCCESS" != "true" ]; then + echo "❌ Upload failed after $MAX_ATTEMPTS attempts" + exit 1 + fi echo "ARTIFACTORY_LAST_BUILD_PATH=${PATH_TO_LATEST_ARTIFACT}" >> $GITHUB_OUTPUT @@ -237,16 +859,24 @@ jobs: packages: write name: Push artifact to ghcr.io - needs: hidrive-next-build + needs: [prepare-matrix, build-apps, hidrive-next-build] + if: | + always() && + (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || + github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' || + startsWith(github.ref_name, 'rc/') || contains(github.ref_name, '/dev/')) && + needs.prepare-matrix.result == 'success' && + (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') && + needs.hidrive-next-build.result == 'success' steps: - name: Download artifact zip - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: hidrive_next_build_artifact - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -254,7 +884,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" @@ -286,18 +916,22 @@ jobs: exit 1 # make it red to grab attention - trigger-remote-dev-worflow: + trigger-remote-dev-workflow: runs-on: self-hosted name: Trigger remote workflow needs: [ hidrive-next-build, upload-to-artifactory ] - # Trigger remote build on "ionos-dev|ionos-stable" branch *push* defined in the on:push:branches - # Can be disabled via repository variable 'DISABLE_REMOTE_TRIGGER' (set to 'true' to disable) + # Trigger remote build on "ionos-dev|ionos-stable|rc/*|*/dev/*" branch *push* defined in on:push:branches + # Can be disabled entirely via repository variable 'DISABLE_REMOTE_TRIGGER' (set to 'true' to disable) + # The "*/dev/*" branch class is gated by repository variable 'ENABLE_REMOTE_TRIGGER_USER_DEV' + # (default off — set to 'true' to enable once GitLab supports BUILD_TYPE=dev-) # Configure at: https://github.com/IONOS-Productivity/nc-server/settings/variables/actions if: | always() && github.event_name == 'push' && - (github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + (github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' || + startsWith(github.ref_name, 'rc/') || + (contains(github.ref_name, '/dev/') && vars.ENABLE_REMOTE_TRIGGER_USER_DEV == 'true')) && needs.hidrive-next-build.result == 'success' && needs.upload-to-artifactory.result == 'success' && vars.DISABLE_REMOTE_TRIGGER != 'true' @@ -358,16 +992,22 @@ jobs: set -x # Branch to GitLab Trigger Mapping (see HDNEXT-1373): - # | ref_name | GITLAB_REF | BUILD_TYPE | - # |--------------|------------|------------| - # | ionos-dev | main | dev | - # | ionos-stable | main | stable | + # | ref_name | GITLAB_REF | BUILD_TYPE | + # |--------------|------------|-------------------| + # | ionos-dev | main | dev | + # | ionos-stable | main | stable | + # | rc/* | main | rc | + # | */dev/* | main | dev- | (opt-in via ENABLE_REMOTE_TRIGGER_USER_DEV) BUILD_TYPE="dev" - # Override build type for stable branch - if [ ${{ github.ref_name }} == "ionos-stable" ]; then + if [ "${{ github.ref_name }}" == "ionos-stable" ]; then BUILD_TYPE="stable" + elif [[ "${{ github.ref_name }}" =~ ^rc/ ]]; then + BUILD_TYPE="rc" + elif [[ "${{ github.ref_name }}" =~ ^.*/dev/.*$ ]]; then + BRANCH_PREFIX=$(echo "${{ github.ref_name }}" | sed 's|/.*||' | sed 's/[^A-Za-z0-9._-]/-/g') + BUILD_TYPE="dev-${BRANCH_PREFIX}" fi # Construct source build URL for traceability diff --git a/.gitmodules b/.gitmodules index 6e47005fa3f6a..762ee80885526 100644 --- a/.gitmodules +++ b/.gitmodules @@ -39,4 +39,4 @@ url = git@github.com:nextcloud/files_downloadlimit.git [submodule "IONOS"] path = IONOS - url = git@github.com:IONOS-Productivity/easystorage-config.git + url = git@github.com:IONOS-Productivity/nc-config.git diff --git a/IONOS b/IONOS index cce9a718b9efe..6dedd722de8b7 160000 --- a/IONOS +++ b/IONOS @@ -1 +1 @@ -Subproject commit cce9a718b9efedcfbc2ed5ef1a967bda7b33d8dd +Subproject commit 6dedd722de8b72ab28da600e0ed0966ab9086bc4