diff --git a/.github/workflows/lint-powershell.yml b/.github/workflows/lint-powershell.yml index 4ae7cd670..0c8890c85 100644 --- a/.github/workflows/lint-powershell.yml +++ b/.github/workflows/lint-powershell.yml @@ -76,6 +76,10 @@ jobs: shell: pwsh run: ./tests/test-windows-model-activation.ps1 + - name: Windows Dotenv Serializer Contract + shell: pwsh + run: ./tests/test-windows-dotenv-serializer.ps1 + - name: Windows Catalog Source Boundary Contract shell: pwsh run: ./tests/test-windows-catalog-selector.ps1 diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index b5defb185..3dc0d8d9d 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -169,6 +169,9 @@ jobs: - name: Safe Env Loading Tests run: bash tests/test-safe-env.sh + - name: Dotenv Serializer Tests + run: bash tests/test-dotenv-serializer.sh + - name: Mode Switch Tests run: bash tests/test-mode-switch-status.sh diff --git a/ods/installers/macos/lib/env-generator.sh b/ods/installers/macos/lib/env-generator.sh index c97d50c88..e812c1cdb 100755 --- a/ods/installers/macos/lib/env-generator.sh +++ b/ods/installers/macos/lib/env-generator.sh @@ -39,6 +39,11 @@ read_env_value() { grep -E "^${key}=" "$env_path" 2>/dev/null | sed -n '1p' | cut -d'=' -f2- | tr -d '\r' || true } +# shellcheck source=../../../lib/dotenv-quote.sh +_ODS_MACOS_ENV_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +. "$_ODS_MACOS_ENV_ROOT/lib/dotenv-quote.sh" +unset _ODS_MACOS_ENV_ROOT + env_key_exists() { local env_path="$1" local key="$2" @@ -535,13 +540,13 @@ CTX_SIZE=${MAX_CONTEXT} MODEL_RECOMMENDED_MODEL=${LLM_MODEL} MODEL_RECOMMENDED_GGUF=${GGUF_FILE} MODEL_RECOMMENDED_CONTEXT=${MAX_CONTEXT} -MODEL_RECOMMENDATION_SOURCE=${MODEL_RECOMMENDATION_SOURCE:-installer_tier_map} -MODEL_RECOMMENDATION_POLICY=${MODEL_RECOMMENDATION_POLICY:-tier-map} -MODEL_RECOMMENDATION_CONFIDENCE=${MODEL_RECOMMENDATION_CONFIDENCE:-medium} -MODEL_RECOMMENDATION_REASON=${MODEL_RECOMMENDATION_REASON:-Selected by installer tier ${tier} (${TIER_NAME}) for apple backend; benchmark locally after first launch.} -MODEL_RECOMMENDED_ALTERNATIVES=${MODEL_RECOMMENDED_ALTERNATIVES:-} +MODEL_RECOMMENDATION_SOURCE=$(dotenv_quote "${MODEL_RECOMMENDATION_SOURCE:-installer_tier_map}") +MODEL_RECOMMENDATION_POLICY=$(dotenv_quote "${MODEL_RECOMMENDATION_POLICY:-tier-map}") +MODEL_RECOMMENDATION_CONFIDENCE=$(dotenv_quote "${MODEL_RECOMMENDATION_CONFIDENCE:-medium}") +MODEL_RECOMMENDATION_REASON=$(dotenv_quote "${MODEL_RECOMMENDATION_REASON:-Selected by installer tier ${tier} (${TIER_NAME}) for apple backend; benchmark locally after first launch.}") +MODEL_RECOMMENDED_ALTERNATIVES=$(dotenv_quote "${MODEL_RECOMMENDED_ALTERNATIVES:-}") MODEL_PERFORMANCE_SOURCE=benchmark_required -MODEL_PERFORMANCE_LABEL=Benchmark after first launch +MODEL_PERFORMANCE_LABEL=$(dotenv_quote "Benchmark after first launch") GPU_BACKEND=apple HOST_RAM_GB=${SYSTEM_RAM_GB} N_GPU_LAYERS=${n_gpu_layers} diff --git a/ods/installers/phases/06-directories.sh b/ods/installers/phases/06-directories.sh index 5f909c8f7..b96437a39 100755 --- a/ods/installers/phases/06-directories.sh +++ b/ods/installers/phases/06-directories.sh @@ -47,6 +47,9 @@ else # shellcheck source=../lib/llama-memory-budget.sh source "$SCRIPT_DIR/installers/lib/llama-memory-budget.sh" + # shellcheck source=../../lib/dotenv-quote.sh + source "$SCRIPT_DIR/lib/dotenv-quote.sh" + _phase06_rootless=false if [[ -f "$SCRIPT_DIR/lib/rootless-ownership.sh" ]]; then # shellcheck source=../../lib/rootless-ownership.sh @@ -825,13 +828,13 @@ CTX_SIZE=${MAX_CONTEXT} MODEL_RECOMMENDED_MODEL=${MODEL_RECOMMENDED_MODEL_VALUE} MODEL_RECOMMENDED_GGUF=${MODEL_RECOMMENDED_GGUF_VALUE} MODEL_RECOMMENDED_CONTEXT=${MODEL_RECOMMENDED_CONTEXT_VALUE} -MODEL_RECOMMENDATION_SOURCE=${MODEL_RECOMMENDATION_SOURCE:-installer_tier_map} -MODEL_RECOMMENDATION_POLICY=${MODEL_RECOMMENDATION_POLICY:-tier-map} -MODEL_RECOMMENDATION_CONFIDENCE=${MODEL_RECOMMENDATION_CONFIDENCE:-medium} -MODEL_RECOMMENDATION_REASON=${MODEL_RECOMMENDATION_REASON:-Selected by installer tier ${TIER} (${TIER_NAME}) for ${GPU_BACKEND} backend; benchmark locally after first launch.} -MODEL_RECOMMENDED_ALTERNATIVES=${MODEL_RECOMMENDED_ALTERNATIVES:-} +MODEL_RECOMMENDATION_SOURCE=$(dotenv_quote "${MODEL_RECOMMENDATION_SOURCE:-installer_tier_map}") +MODEL_RECOMMENDATION_POLICY=$(dotenv_quote "${MODEL_RECOMMENDATION_POLICY:-tier-map}") +MODEL_RECOMMENDATION_CONFIDENCE=$(dotenv_quote "${MODEL_RECOMMENDATION_CONFIDENCE:-medium}") +MODEL_RECOMMENDATION_REASON=$(dotenv_quote "${MODEL_RECOMMENDATION_REASON:-Selected by installer tier ${TIER} (${TIER_NAME}) for ${GPU_BACKEND} backend; benchmark locally after first launch.}") +MODEL_RECOMMENDED_ALTERNATIVES=$(dotenv_quote "${MODEL_RECOMMENDED_ALTERNATIVES:-}") MODEL_PERFORMANCE_SOURCE=benchmark_required -MODEL_PERFORMANCE_LABEL=Benchmark after first launch +MODEL_PERFORMANCE_LABEL=$(dotenv_quote "Benchmark after first launch") GPU_BACKEND=${GPU_BACKEND} SYSTEM_RAM_GB=${RAM_GB:-0} N_GPU_LAYERS=${N_GPU_LAYERS_VALUE} diff --git a/ods/installers/windows/lib/env-generator.ps1 b/ods/installers/windows/lib/env-generator.ps1 index a667748a5..47ee28d75 100644 --- a/ods/installers/windows/lib/env-generator.ps1 +++ b/ods/installers/windows/lib/env-generator.ps1 @@ -457,6 +457,22 @@ function New-SecureBase64 { return [Convert]::ToBase64String($buf) } +function ConvertTo-ODSDotenvValue { + <# Serialize a value for Bash, Docker Compose, and ODS safe-env readers. #> + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + + $text = ([string]$Value) -replace "`r", " " -replace "`n", " " + if ($text.IndexOf("'") -ge 0) { + # Bash and Compose disagree about \` inside double-quoted dotenv + # values. Normalize it only in this apostrophe fallback so both readers + # receive the same safe text. + $text = $text.Replace('`', 'ˋ') + $escaped = $text.Replace('\', '\\').Replace('"', '\"').Replace('$', '\$') + return '"' + $escaped + '"' + } + return "'" + $text + "'" +} + function New-ODSEnv { <# .SYNOPSIS @@ -876,6 +892,13 @@ function New-ODSEnv { if ([string]::IsNullOrWhiteSpace($nGpuLayers)) { $nGpuLayers = "auto" } # Build .env content (matches Phase 06 format) + $recommendationSource = ConvertTo-ODSDotenvValue $(if ($TierConfig.RecommendationSource) { $TierConfig.RecommendationSource } else { "installer_tier_map" }) + $recommendationPolicy = ConvertTo-ODSDotenvValue $(if ($TierConfig.RecommendationPolicy) { $TierConfig.RecommendationPolicy } else { "tier-map" }) + $recommendationConfidence = ConvertTo-ODSDotenvValue $(if ($TierConfig.RecommendationConfidence) { $TierConfig.RecommendationConfidence } else { "medium" }) + $recommendationReason = ConvertTo-ODSDotenvValue $(if ($TierConfig.RecommendationReason) { $TierConfig.RecommendationReason } else { "Selected by installer tier $Tier ($($TierConfig.TierName)) for $GpuBackend backend; benchmark locally after first launch." }) + $recommendationAlternatives = ConvertTo-ODSDotenvValue $(if ($TierConfig.RecommendationAlternatives) { $TierConfig.RecommendationAlternatives } else { "" }) + $performanceLabel = ConvertTo-ODSDotenvValue "Benchmark after first launch" + $envContent = @" # ODS Configuration -- $($TierConfig.TierName) Edition # Generated by Windows installer v$($script:ODS_VERSION) on $timestamp @@ -923,16 +946,16 @@ CTX_SIZE=$($TierConfig.MaxContext) MODEL_RECOMMENDED_MODEL=$($TierConfig.LlmModel) MODEL_RECOMMENDED_GGUF=$($TierConfig.GgufFile) MODEL_RECOMMENDED_CONTEXT=$($TierConfig.MaxContext) -MODEL_RECOMMENDATION_SOURCE=$(if ($TierConfig.RecommendationSource) { $TierConfig.RecommendationSource } else { "installer_tier_map" }) -MODEL_RECOMMENDATION_POLICY=$(if ($TierConfig.RecommendationPolicy) { $TierConfig.RecommendationPolicy } else { "tier-map" }) -MODEL_RECOMMENDATION_CONFIDENCE=$(if ($TierConfig.RecommendationConfidence) { $TierConfig.RecommendationConfidence } else { "medium" }) -MODEL_RECOMMENDATION_REASON=$(if ($TierConfig.RecommendationReason) { $TierConfig.RecommendationReason } else { "Selected by installer tier $Tier ($($TierConfig.TierName)) for $GpuBackend backend; benchmark locally after first launch." }) -MODEL_RECOMMENDED_ALTERNATIVES=$(if ($TierConfig.RecommendationAlternatives) { $TierConfig.RecommendationAlternatives } else { "" }) +MODEL_RECOMMENDATION_SOURCE=$recommendationSource +MODEL_RECOMMENDATION_POLICY=$recommendationPolicy +MODEL_RECOMMENDATION_CONFIDENCE=$recommendationConfidence +MODEL_RECOMMENDATION_REASON=$recommendationReason +MODEL_RECOMMENDED_ALTERNATIVES=$recommendationAlternatives MODEL_RUNTIME_PROFILE=$(if ($TierConfig.RuntimeProfile) { $TierConfig.RuntimeProfile } else { "" }) MODEL_RUNTIME_PROFILE_LABEL=$(if ($TierConfig.RuntimeProfileLabel) { $TierConfig.RuntimeProfileLabel } else { "" }) MODEL_RUNTIME_PROFILE_SOURCE=$(if ($TierConfig.RuntimeProfileSource) { $TierConfig.RuntimeProfileSource } else { "" }) MODEL_PERFORMANCE_SOURCE=benchmark_required -MODEL_PERFORMANCE_LABEL=Benchmark after first launch +MODEL_PERFORMANCE_LABEL=$performanceLabel GPU_BACKEND=$GpuBackend SYSTEM_RAM_GB=$SystemRamGB N_GPU_LAYERS=$nGpuLayers diff --git a/ods/lib/dotenv-quote.sh b/ods/lib/dotenv-quote.sh new file mode 100755 index 000000000..212bbb593 --- /dev/null +++ b/ods/lib/dotenv-quote.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Serialize one value for the single-line .env grammar shared by Bash, Docker +# Compose, and ODS's safe environment reader. Newlines cannot be represented +# portably in that grammar, so normalize them to spaces rather than joining +# words or allowing a second assignment line. +dotenv_quote() { + local value="$1" + value="${value//$'\r'/ }" + value="${value//$'\n'/ }" + + # Single quotes are literal in both Bash and Compose. When the value itself + # contains one, use their common double-quoted escape set. Bash requires a + # backslash before a literal backtick there, while Compose preserves that + # backslash, so normalize backticks to the visually equivalent modifier + # grave accent only in this rare fallback instead of corrupting one reader. + if [[ "$value" == *"'"* ]]; then + value="${value//\`/ˋ}" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="${value//\$/\\\$}" + printf '"%s"\n' "$value" + else + printf "'%s'\n" "$value" + fi +} diff --git a/ods/lib/safe-env.sh b/ods/lib/safe-env.sh index f1e079fd4..a85f0c1b9 100755 --- a/ods/lib/safe-env.sh +++ b/ods/lib/safe-env.sh @@ -48,6 +48,7 @@ load_env_file() { if [[ "$value" == '"'*'"' ]]; then value="${value#\"}" value="${value%\"}" + value="$(_safe_env_unescape_double_quoted "$value")" elif [[ "$value" == "'"*"'" ]]; then value="${value#\'}" value="${value%\'}" diff --git a/ods/tests/smoke/installer-env-smoke.sh b/ods/tests/smoke/installer-env-smoke.sh index eebc45573..47cdf3424 100755 --- a/ods/tests/smoke/installer-env-smoke.sh +++ b/ods/tests/smoke/installer-env-smoke.sh @@ -125,6 +125,8 @@ if bash -c " export LLM_MODEL=qwen3-1.7b export GGUF_FILE=Qwen3-1.7B-Q4_K_M.gguf export MAX_CONTEXT=4096 + export MODEL_RECOMMENDATION_REASON='Arch-aware catalog policy (spark-aarch64): selected after fit check' + export MODEL_RECOMMENDED_ALTERNATIVES='deepseek-r1:32768:48;qwen-a3b:131072:35.48' export ODS_VERSION=2.1.0 export ENABLE_VOICE=true export ENABLE_WORKFLOWS=true @@ -186,6 +188,19 @@ fi echo "" echo "── .env schema validation ──" if [[ "$ENV_GENERATED" == true && -f "$INSTALL_DIR/.env" ]]; then + if bash -c ' + set -a + source "$1" + set +a + [[ "$MODEL_RECOMMENDATION_REASON" == "Arch-aware catalog policy (spark-aarch64): selected after fit check" ]] + [[ "$MODEL_RECOMMENDED_ALTERNATIVES" == "deepseek-r1:32768:48;qwen-a3b:131072:35.48" ]] + [[ "$MODEL_PERFORMANCE_LABEL" == "Benchmark after first launch" ]] + ' _ "$INSTALL_DIR/.env"; then + pass "Generated model metadata is safe to source as dotenv" + else + fail "Generated model metadata breaks dotenv parsing" + fi + if bash scripts/validate-env.sh "$INSTALL_DIR/.env" "$ROOT_DIR/.env.schema.json" 2>/dev/null; then pass ".env validates against schema" else diff --git a/ods/tests/test-dotenv-serializer.sh b/ods/tests/test-dotenv-serializer.sh new file mode 100755 index 000000000..b9d123614 --- /dev/null +++ b/ods/tests/test-dotenv-serializer.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -u + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +. "$ROOT_DIR/lib/dotenv-quote.sh" +. "$ROOT_DIR/lib/safe-env.sh" + +pass_count=0 +fail_count=0 +pass() { pass_count=$((pass_count + 1)); } +fail() { printf 'FAIL: %s\n' "$1" >&2; fail_count=$((fail_count + 1)); } + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +values=( + "Arch-aware catalog policy (spark-aarch64): selected after fit check" + "deepseek-r1:32768:48;qwen-a3b:131072:35.48" + 'contains "literal double quotes"' + 'cost is $HOME and ${UNSET_VAR} and $((1+1))' + 'command $(touch pwned) substitution' + 'backtick `id` here' + 'C:\Users\dev\ods\models' + $'carriage\rreturn' + $'multi\nline' + "" + "it's a model" + 'it'"'"'s $HOME $(touch pwned) `id` C:\path "dq"' +) + +for value in "${values[@]}"; do + quoted="$(dotenv_quote "$value")" + expected="${value//$'\r'/ }" + expected="${expected//$'\n'/ }" + if [[ "$expected" == *"'"* ]]; then + expected="${expected//\`/ˋ}" + fi + printf 'TESTVAR=%s\n' "$quoted" > "$tmp_dir/.env" + + unset TESTVAR + actual="$(set -a; source "$tmp_dir/.env"; set +a; printf '%s' "${TESTVAR-}")" + [[ "$actual" == "$expected" ]] \ + && pass \ + || fail "Bash source did not round-trip [$value]" + + unset TESTVAR + load_env_file "$tmp_dir/.env" + [[ "${TESTVAR-}" == "$expected" ]] \ + && pass \ + || fail "safe-env did not round-trip [$value]" +done + +[[ ! -e "$tmp_dir/pwned" ]] \ + && pass \ + || fail "serialized command substitution executed" +[[ "$(dotenv_quote 'a;b $HOME')" == "$(dotenv_quote 'a;b $HOME')" ]] \ + && pass \ + || fail "serialization is not deterministic" + +# Compose has its own dotenv reader. Parsing every adversarial value proves the +# generated file remains valid there without requiring an image pull in CI. +if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + : > "$tmp_dir/compose.env" + { + printf '%s\n' 'services:' ' probe:' ' image: busybox' ' environment:' + index=0 + for value in "${values[@]}"; do + index=$((index + 1)) + printf 'CASE%d=%s\n' "$index" "$(dotenv_quote "$value")" >> "$tmp_dir/compose.env" + printf ' CASE%d: ${CASE%d}\n' "$index" "$index" + done + } > "$tmp_dir/compose.yaml" + docker compose --env-file "$tmp_dir/compose.env" -f "$tmp_dir/compose.yaml" config >/dev/null 2>&1 \ + && pass \ + || fail "Docker Compose rejected serialized values" +fi + +printf 'Results: %d passed, %d failed\n' "$pass_count" "$fail_count" +[[ "$fail_count" -eq 0 ]] diff --git a/ods/tests/test-safe-env.sh b/ods/tests/test-safe-env.sh index 579c97c8f..873f18baa 100755 --- a/ods/tests/test-safe-env.sh +++ b/ods/tests/test-safe-env.sh @@ -141,5 +141,15 @@ load_env_file "$tmpdir/.env-quotes" [[ "${PLAIN_SQ:-}" == "plain" ]] || fail "PLAIN_SQ not unquoted (got: '${PLAIN_SQ:-}')" pass "load_env_file strips only matched surrounding quote pairs" +echo "Test 13: load_env_file decodes the supported double-quoted escape set" +unset FILE_ESCAPED 2>/dev/null || true +cat > "$tmpdir/.env-escaped" << 'EOF' +FILE_ESCAPED="it's \$HOME and \$(whoami) and \`id\` and C:\\path and \"dq\"" +EOF +load_env_file "$tmpdir/.env-escaped" +[[ "${FILE_ESCAPED:-}" == 'it'"'"'s $HOME and $(whoami) and `id` and C:\path and "dq"' ]] \ + || fail "FILE_ESCAPED not decoded safely (got: ${FILE_ESCAPED:-})" +pass "load_env_file decodes supported escapes without evaluation" + echo "" echo "All safe-env tests passed." diff --git a/ods/tests/test-windows-dotenv-serializer.ps1 b/ods/tests/test-windows-dotenv-serializer.ps1 new file mode 100644 index 000000000..18eb43551 --- /dev/null +++ b/ods/tests/test-windows-dotenv-serializer.ps1 @@ -0,0 +1,30 @@ +$ErrorActionPreference = "Stop" + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +. (Join-Path $repoRoot "installers/windows/lib/env-generator.ps1") + +$failures = 0 +function Assert-Equal([string]$Label, [string]$Expected, [string]$Actual) { + if ($Expected -ne $Actual) { + Write-Host "FAIL: $Label expected=[$Expected] actual=[$Actual]" + $script:failures++ + } else { + Write-Host "PASS: $Label" + } +} + +$simple = 'deepseek-r1:32768:48;qwen-a3b:131072:35.48' +Assert-Equal "simple value" "'$simple'" (ConvertTo-ODSDotenvValue $simple) + +$special = 'cost is $HOME and $(whoami) and `id` and "dq" and C:\path' +Assert-Equal "literal special characters" "'$special'" (ConvertTo-ODSDotenvValue $special) + +$compound = 'it''s $HOME and $(whoami) and `id` and "dq" and C:\path' +$compoundExpected = '"it''s \$HOME and \$(whoami) and ˋidˋ and \"dq\" and C:\\path"' +Assert-Equal "single quote fallback" $compoundExpected (ConvertTo-ODSDotenvValue $compound) + +Assert-Equal "empty value" "''" (ConvertTo-ODSDotenvValue "") +Assert-Equal "line normalization" "'line break'" (ConvertTo-ODSDotenvValue "line`nbreak") +Assert-Equal "deterministic" (ConvertTo-ODSDotenvValue 'a;b $HOME') (ConvertTo-ODSDotenvValue 'a;b $HOME') + +if ($failures -gt 0) { exit 1 }