diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 577e755d..91fe8493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ # Fast, BLOCKING CI. Runs on every push and pull request. # # Everything provable on a plain Linux box with no Xcode and no Android SDK: -# lint, format, typecheck, build, the full vitest unit suite (all four +# lint, format, typecheck, build, the full vitest unit suite (all five # packages), and the fast cross-platform e2e (test/e2e/cache-flow.e2e.js), which # drives the real CLI and the real cache library end to end with no compiler. # The SLOW native builds live in e2e-native.yml, gated so a flaky 15-minute @@ -72,8 +72,8 @@ jobs: - name: Dead code (knip) run: pnpm run knip - # The unit suite on vitest (rolldown-vite): stim-cli + both cache packages, - # 52 files. Real processes, real ports, real git; fakes for xcrun/adb. + # The unit suite on vitest (rolldown-vite): stim-cli + the cache packages, + # 70 files. Real processes, real ports, real git; fakes for xcrun/adb. - name: Unit tests (vitest) run: pnpm test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 70f7b7a1..db0bac97 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# Publishes the four packages to npm via OIDC trusted publishing when a +# Publishes the five packages to npm via OIDC trusted publishing when a # version tag is pushed. There is NO token: npm trusts a short-lived OIDC # credential minted for exactly this repo + workflow + environment, and the # `release` environment requires a manual approval, which replaces the OTP. @@ -44,7 +44,7 @@ jobs: - name: Tag matches the package versions run: | tag="${GITHUB_REF_NAME#v}" - for p in core stim-cli expo-build-cache metro; do + for p in core cache stim-cli expo-build-cache metro; do v=$(node -p "require('./packages/$p/package.json').version") if [ "$v" != "$tag" ]; then echo "packages/$p is $v but the tag is $tag"; exit 1 @@ -59,6 +59,15 @@ jobs: else npm publish --provenance --access public --tag latest fi + - name: Publish @stim-cli/cache + working-directory: packages/cache + run: | + version=$(node -p "require('./package.json').version") + if npm view "@stim-cli/cache@$version" version >/dev/null 2>&1; then + echo "@stim-cli/cache@$version is already published" + else + npm publish --provenance --access public --tag latest + fi - name: Publish @stim-cli/metro working-directory: packages/metro run: | @@ -91,7 +100,7 @@ jobs: version="${GITHUB_REF_NAME#v}" max_attempts=12 retry_delay=10 - for p in @stim-cli/core stim-cli @stim-cli/expo-build-cache @stim-cli/metro; do + for p in @stim-cli/core @stim-cli/cache stim-cli @stim-cli/expo-build-cache @stim-cli/metro; do for attempt in $(seq 1 "$max_attempts"); do if v=$(npm view "$p@$version" version 2>/dev/null) && [ "$v" = "$version" ]; then echo "$p -> $v" diff --git a/README.md b/README.md index d5ea5ffd..bab65b10 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ stim guide settings - [`@stim-cli/metro`](./packages/metro) shares Metro transforms and records logs. - [`@stim-cli/expo-build-cache`](./packages/expo-build-cache) lets direct Expo builds share native artifacts with Stim. +- [`@stim-cli/cache`](./packages/cache) holds the cache provider contract and + the local-first tier coordination behind both caches. - [`@stim-cli/core`](./packages/core) contains shared internal cache contracts. ## Development diff --git a/RELEASE.md b/RELEASE.md index f83601ae..defda92a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -4,11 +4,12 @@ How to cut a new version of `stim-cli` to npm and GitHub. Keep this in sync with what we actually do — when something changes, update both this file and the real workflow at the same time. -## 0. The four packages +## 0. The five packages ``` packages/core @stim-cli/core shared primitives (cache roots, cache key, registration) packages/stim-cli stim-cli the CLI +packages/cache @stim-cli/cache cache provider contract and tier coordination packages/expo-build-cache @stim-cli/expo-build-cache Expo build cache provider packages/metro @stim-cli/metro shared Metro transform cache + log reporter ``` @@ -47,7 +48,7 @@ fi ``` An npm `E404` means this is the first release for that package name. Confirm -all four names are available, use the intended first version, and review the +all five names are available, use the intended first version, and review the full release diff. Complete the first-publication bootstrap in section 4, step 7 before pushing the first tag. @@ -83,7 +84,7 @@ Start from `main`, fully up to date with `origin/main`. Before candidate preparation, `git status --short` may show only the draft `docs/releases/X.Y.Z.md`. -1. **Bump the version in lockstep.** All four `package.json` files carry the +1. **Bump the version in lockstep.** All five `package.json` files carry the same number, and `dist/cli.mjs` reads it from its own `package.json`: ```bash @@ -91,16 +92,16 @@ preparation, `git status --short` may show only the draft pnpm install --lockfile-only ``` - The filtered `exec` bumps all four; `--no-git-tag-version` leaves the + The filtered `exec` bumps all five; `--no-git-tag-version` leaves the candidate uncommitted and untagged. The lockfile duplicates every workspace's version and must move with the manifests. - Confirm all four moved, and that dependency ranges between the packages + Confirm all five moved, and that dependency ranges between the packages still name versions that will exist when publishing finishes: ```bash grep -H '"version"' packages/*/package.json - grep -H '"@stim-cli/' packages/stim-cli/package.json packages/expo-build-cache/package.json packages/metro/package.json + grep -H '"@stim-cli/' packages/stim-cli/package.json packages/expo-build-cache/package.json packages/metro/package.json packages/cache/package.json ``` 2. **Install and run the full pre-flight against those exact files:** @@ -133,7 +134,7 @@ preparation, `git status --short` may show only the draft packages). Every published JavaScript entry lives under `dist/`. 4. **Inspect the candidate diff.** `git status --short` should contain only the - four package manifests, `pnpm-lock.yaml`, and the draft + five package manifests, `pnpm-lock.yaml`, and the draft release notes. Resolve anything else before QA. ## 3. Pre-tag QA gate @@ -215,7 +216,7 @@ Before continuing: ``` One tag for the repo, not one per package: the packages share a version, so - a per-package tag would only say the same thing four times. + a per-package tag would only say the same thing five times. 6. **Publish the already-reviewed release notes in `docs/releases/X.Y.Z.md`.** This committed file is the single source of @@ -248,18 +249,19 @@ Before continuing: provenance publish is REJECTED without it, E422), and a NEW package must be published once by hand first -- npm's trusted-publisher settings live on the package page, which does not exist until then. For the first - `stim-cli` release, create the `@stim-cli` npm organization, publish all four + `stim-cli` release, create the `@stim-cli` npm organization, publish all five packages manually in dependency order, then configure each package's trusted publisher for `appandflow/stim`, workflow `release.yml`, environment `release`. Do this before pushing the first tag. The tagged workflow skips an exact package version that already exists, uses the npm `next` dist-tag for - prereleases, then verifies all four registry versions. The same commands are + prereleases, then verifies all five registry versions. The same commands are the manual fallback for later releases. Add `--tag next` to every command when publishing a prerelease: ```bash npm whoami # confirm login; if 401, `npm login` first pnpm --filter @stim-cli/core publish --access public --otp + pnpm --filter @stim-cli/cache publish --access public --otp pnpm --filter @stim-cli/metro publish --access public --otp pnpm --filter @stim-cli/expo-build-cache publish --access public --otp pnpm --filter stim-cli publish --access public --otp @@ -278,6 +280,7 @@ Before continuing: version=X.Y.Z cd /tmp && npx "stim-cli@$version" --version npm view "stim-cli@$version" readme | head -c 200 # NOT "No README data found!" + npm view "@stim-cli/cache@$version" version npm view "@stim-cli/expo-build-cache@$version" version npm view "@stim-cli/metro@$version" version ``` diff --git a/docs/e2e-and-ci.md b/docs/e2e-and-ci.md index 0e627e48..f1680ede 100644 --- a/docs/e2e-and-ci.md +++ b/docs/e2e-and-ci.md @@ -6,7 +6,7 @@ default `~/.stim/workspaces/...`). Stim does not create a project `.gitignore` entry for this state. Stim has three test layers. The unit suite (`pnpm test`, Vitest, more than -2,000 cases across four packages) is the bulk of the coverage. On top of it sit +2,000 cases across five packages) is the bulk of the coverage. On top of it sit two end-to-end layers that exercise the _published loop_ rather than individual functions. The separately built runtime-floor job loads every published ESM entry point on Node 20.19.4. diff --git a/package.json b/package.json index b5c9e93c..3ac5e46a 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ ], "type": "module", "scripts": { - "typecheck": "tsc --noEmit -p packages/core/tsconfig.json && tsc --noEmit -p packages/stim-cli/tsconfig.json && tsc --noEmit -p packages/metro/tsconfig.json && tsc --noEmit -p packages/expo-build-cache/tsconfig.json", + "typecheck": "tsc --noEmit -p packages/core/tsconfig.json && tsc --noEmit -p packages/cache/tsconfig.json && tsc --noEmit -p packages/stim-cli/tsconfig.json && tsc --noEmit -p packages/metro/tsconfig.json && tsc --noEmit -p packages/expo-build-cache/tsconfig.json", "lint": "oxlint .", "lint:fix": "oxlint --fix .", "format": "oxfmt .", diff --git a/packages/cache/LICENSE b/packages/cache/LICENSE new file mode 100644 index 00000000..021991ea --- /dev/null +++ b/packages/cache/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Janic Duplessis + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/cache/README.md b/packages/cache/README.md new file mode 100644 index 00000000..d097807d --- /dev/null +++ b/packages/cache/README.md @@ -0,0 +1,141 @@ +# @stim-cli/cache + +The cache provider contract Stim uses for Metro transforms and native build +artifacts, plus the tier coordination, timeout, and warning policy that sit in +front of it. + +The local filesystem is always the first tier. A project can add one optional +second tier by pointing `cache.provider` at a module that implements this +contract. Stim ships no network provider; writing one is an addition, not a +change to Stim. + +If Stim is not installed globally, replace `stim` with `npx stim-cli`. + +## Selecting a provider + +```json +{ + "cache": { + "provider": "./tools/cache-provider.cjs", + "options": { "bucket": "mobile-cache" } + } +} +``` + +The reference is a package name or a path relative to the settings file that +declares it. Machine settings override committed `.stim.json` settings, and the +existing nested merge rules apply to `cache.options`. Keep secrets out of +committed settings; read them from the environment or from machine settings. + +## Writing a provider + +```js +export const apiVersion = 1; + +export async function createCacheProvider({ projectRoot, options }) { + return { + metro: { + async get({ key, cacheName, signal }) { + return null; + }, + async set({ key, value, cacheName, signal }) {}, + }, + builds: { + async resolve({ platform, key, destinationDir, signal }) { + return null; + }, + async store({ platform, key, sourcePath, overwrite, signal }) {}, + }, + }; +} +``` + +A provider implements one or both capabilities. It owns transport, +serialization, archive format, authentication, and remote retention. Stim owns +fingerprints, cache keys, and local artifact paths. + +`metro.get` returns the stored value or `null`. + +`builds.resolve` returns an existing path to the artifact, or `null` for a +miss. `destinationDir` is a scratch directory Stim creates and owns: a provider +that fetches the artifact must materialize it there and return a path inside +it, and must leave the directory empty on a miss. The built-in filesystem tier +already holds the artifact, so it returns its own cache path instead. + +`builds.store` receives the `.app` directory or `.apk` file that Stim just +built. `overwrite: false` must keep an entry that already exists for the key, +and `overwrite: true` must replace it. + +Every call receives an `AbortSignal`. A provider must honor it: Stim abandons +the call at the deadline and keeps building or bundling with the local tier. + +`stim gc` never deletes provider data, and the contract has no delete +operation, so shared team or CI data is never removed by a local command. + +## Failure rules + +Provider failures are cache misses. A timeout, module error, authentication +error, or network error produces one warning per failure class per command or +supervisor run and never fails a bundle, an install, a launch, or a successful +build. + +## Contract tests + +Run the shipped checks against your own module: + +```js +import { runCacheProviderContract } from '@stim-cli/cache'; + +const results = await runCacheProviderContract({ + provider: await createCacheProvider({ projectRoot, options }), + projectRoot, + workDir, +}); + +for (const result of results) { + if (!result.passed) throw new Error(`${result.name}: ${result.error}`); +} +``` + +Pass `providerModule` instead of `provider` to load the module the way Stim +does, which also checks `apiVersion` and the factory: + +```js +const results = await runCacheProviderContract({ + providerModule: './tools/cache-provider.cjs', + projectRoot, + workDir, +}); +``` + +`cacheProviderContractChecks()` returns the same checks as individual cases for +a test runner that reports each one separately. Both helpers only check the +capabilities a provider advertises, bound every check with a deadline, and +verify that a call settles once its `AbortSignal` aborts. + +## Budgets + +Every provider call is bounded. The defaults are 2s for a Metro read, 10s for a +Metro write, 30s for a build lookup, 60s for a build upload, and 10s to load the +module. Override any of them per run with an environment variable: + +```bash +STIM_CACHE_METRO_READ_TIMEOUT_MS=5000 stim start +STIM_CACHE_BUILD_RESOLVE_TIMEOUT_MS=60000 stim ios +``` + +The variables are `STIM_CACHE_METRO_READ_TIMEOUT_MS`, +`STIM_CACHE_METRO_WRITE_TIMEOUT_MS`, `STIM_CACHE_BUILD_RESOLVE_TIMEOUT_MS`, +`STIM_CACHE_BUILD_UPLOAD_TIMEOUT_MS`, and `STIM_CACHE_LOAD_TIMEOUT_MS`. Each +takes whole milliseconds; any other value keeps the default. + +The tiered Metro store also exposes `flush()`, which resolves once queued +provider writes have drained. Metro never calls it; every in-flight write holds +a referenced deadline, so a Metro process drains on its own within the write +budget. It exists for tests and embedders that own the process. + +Metro reads are also capped: at most six can be in flight, and the tier turns +itself off for the rest of the run after five consecutive failures, so a broken +provider costs one round of warnings rather than a timeout per transform. + +The npm scope remains `@stim-cli` until the `@stim` scope is available. diff --git a/packages/cache/__tests__/builds.test.ts b/packages/cache/__tests__/builds.test.ts new file mode 100644 index 00000000..75ff715d --- /dev/null +++ b/packages/cache/__tests__/builds.test.ts @@ -0,0 +1,349 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + BUILD_RESOLVE_TIMEOUT_ENV, + BUILD_RESOLVE_TIMEOUT_MS, + BUILD_UPLOAD_TIMEOUT_ENV, + BUILD_UPLOAD_TIMEOUT_MS, + buildResolveTimeoutMs, + buildUploadTimeoutMs, + resolveTieredBuild, + storeTieredBuild, +} from '../builds.ts'; +import type { BuildCacheCapability, BuildCacheTarget } from '../provider.ts'; + +let workDir: string; + +beforeEach(() => { + workDir = mkdtempSync(join(tmpdir(), 'stim-cache-builds-')); +}); + +afterEach(() => { + rmSync(workDir, { recursive: true, force: true }); +}); + +function artifact(name: string): string { + const dir = join(workDir, name); + mkdirSync(dir, { recursive: true }); + const path = join(dir, 'app.apk'); + writeFileSync(path, name); + return path; +} + +const target: BuildCacheTarget = { projectRoot: '/repo/app', platform: 'android', key: 'fingerprint-debug-sim' }; + +function capability(overrides: Partial = {}) { + const calls = { resolve: 0, store: 0 }; + const inputs: { resolve: unknown[]; store: unknown[] } = { resolve: [], store: [] }; + const cap: BuildCacheCapability = { + resolve: (input) => { + calls.resolve += 1; + inputs.resolve.push(input); + return overrides.resolve ? overrides.resolve(input) : null; + }, + store: (input) => { + calls.store += 1; + inputs.store.push(input); + return overrides.store ? overrides.store(input) : undefined; + }, + }; + return { cap, calls, inputs }; +} + +test('the pinned build timeouts stay put', () => { + expect(BUILD_RESOLVE_TIMEOUT_MS).toBe(30_000); + expect(BUILD_UPLOAD_TIMEOUT_MS).toBe(60_000); +}); + +test('a local hit does not call the provider', async () => { + const local = capability({ resolve: () => '/cache/android/key/app.apk' }); + const provider = capability(); + + const found = await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ name: './cache.cjs', provider: { builds: provider.cap } }), + target, + destinationDir: workDir, + }); + + expect(found).toEqual({ path: '/cache/android/key/app.apk', tier: 'local' }); + expect(provider.calls.resolve).toBe(0); +}); + +test('a provider hit is stored locally and reports the provider tier', async () => { + const downloaded = artifact('downloaded'); + const stored = artifact('stored'); + const local = capability({ resolve: () => null, store: () => stored }); + const provider = capability({ resolve: () => downloaded }); + + const found = await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ name: './cache.cjs', provider: { builds: provider.cap } }), + target, + destinationDir: workDir, + }); + + expect(found).toEqual({ path: stored, tier: 'provider', providerName: './cache.cjs', storedLocally: true }); + expect(local.inputs.store[0]).toMatchObject({ sourcePath: downloaded, overwrite: false, key: target.key }); + expect(provider.inputs.resolve[0]).toMatchObject({ destinationDir: workDir, key: target.key, platform: 'android' }); +}); + +test('a local backfill failure still returns the downloaded artifact', async () => { + const downloaded = artifact('downloaded'); + const local = capability({ + resolve: () => null, + store: () => { + throw new Error('disk full'); + }, + }); + const provider = capability({ resolve: () => downloaded }); + const warnings: string[] = []; + + const found = await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ name: 'team-cache', provider: { builds: provider.cap } }), + target, + destinationDir: workDir, + warn: (code, message) => warnings.push(`${code}: ${message}`), + }); + + expect(found).toEqual({ path: downloaded, tier: 'provider', providerName: 'team-cache', storedLocally: false }); + expect(warnings[0]).toMatch(/provider-backfill: a team-cache hit could not be stored locally: disk full/); +}); + +test('skipRead bypasses both tiers', async () => { + const local = capability({ resolve: () => '/cache/hit.apk' }); + const provider = capability({ resolve: () => '/remote/hit.apk' }); + + expect( + await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ provider: { builds: provider.cap } }), + target, + destinationDir: workDir, + skipRead: true, + }), + ).toBeNull(); + expect(local.calls.resolve).toBe(0); + expect(provider.calls.resolve).toBe(0); +}); + +test('a provider miss, timeout, failure, or missing path returns a miss', async () => { + const local = capability({ resolve: () => null }); + + const miss = await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ provider: { builds: capability({ resolve: () => null }).cap } }), + target, + destinationDir: workDir, + }); + expect(miss).toBeNull(); + + const timedOutWarnings: string[] = []; + const timedOut = await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ + name: 'team-cache', + provider: { builds: capability({ resolve: () => new Promise(() => {}) }).cap }, + }), + target, + destinationDir: workDir, + timeoutMs: 5, + warn: (_code, message) => timedOutWarnings.push(message), + }); + expect(timedOut).toBeNull(); + expect(timedOutWarnings[0]).toBe('team-cache did not answer within 5ms; building instead'); + + const failureWarnings: string[] = []; + const failed = await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ + name: 'team-cache', + provider: { + builds: capability({ + resolve: () => { + throw new Error('unauthorized'); + }, + }).cap, + }, + }), + target, + destinationDir: workDir, + warn: (_code, message) => failureWarnings.push(message), + }); + expect(failed).toBeNull(); + expect(failureWarnings[0]).toBe('team-cache could not be used: unauthorized; building instead'); + + const missingWarnings: string[] = []; + const missing = await resolveTieredBuild({ + local: local.cap, + loadProvider: () => ({ + name: 'team-cache', + provider: { builds: capability({ resolve: () => join(workDir, 'nope.apk') }).cap }, + }), + target, + destinationDir: workDir, + warn: (_code, message) => missingWarnings.push(message), + }); + expect(missing).toBeNull(); + expect(missingWarnings[0]).toMatch(/which does not exist/); +}); + +test('a fresh build stores locally before the provider upload starts', async () => { + const built = artifact('built'); + const stored = artifact('stored'); + const order: string[] = []; + const local = capability({ + store: () => { + order.push('local'); + return stored; + }, + }); + const provider = capability({ + store: async () => { + order.push('provider'); + }, + }); + + const result = await storeTieredBuild({ + local: local.cap, + loadProvider: () => ({ provider: { builds: provider.cap } }), + target, + sourcePath: built, + overwrite: false, + }); + + expect(result.localPath).toBe(stored); + expect(order).toEqual(['local', 'provider']); + expect(await result.providerUpload).toEqual({ value: undefined }); + expect(local.inputs.store[0]).toMatchObject({ sourcePath: built, overwrite: false }); +}); + +test('a provider upload failure or timeout is reported without throwing', async () => { + const built = artifact('built'); + const local = capability({ store: () => built }); + + const failed = await storeTieredBuild({ + local: local.cap, + loadProvider: () => ({ + provider: { + builds: capability({ + store: () => { + throw new Error('upload denied'); + }, + }).cap, + }, + }), + target, + sourcePath: built, + overwrite: true, + }); + expect(await failed.providerUpload).toEqual({ failed: 'upload denied' }); + + const timedOut = await storeTieredBuild({ + local: local.cap, + loadProvider: () => ({ provider: { builds: capability({ store: () => new Promise(() => {}) }).cap } }), + target, + sourcePath: built, + overwrite: true, + timeoutMs: 5, + }); + expect(await timedOut.providerUpload).toEqual({ timedOut: true }); +}); + +test('without a provider the store reports the local path and no upload', async () => { + const built = artifact('built'); + const local = capability({ store: () => built }); + + const result = await storeTieredBuild({ local: local.cap, target, sourcePath: built, overwrite: false }); + expect(result).toEqual({ localPath: built, providerUpload: null, providerName: null }); +}); + +test('an unusable provider warns once and keeps the local tier', async () => { + const built = artifact('built'); + const local = capability({ resolve: () => null, store: () => built }); + const warnings: string[] = []; + const loadProvider = () => ({ name: './cache.cjs', unavailable: 'missing credentials' }); + + const found = await resolveTieredBuild({ + local: local.cap, + loadProvider, + target, + destinationDir: workDir, + warn: (code, message) => warnings.push(`${code}: ${message}`), + }); + const stored = await storeTieredBuild({ + local: local.cap, + loadProvider, + target, + sourcePath: built, + overwrite: false, + warn: (code, message) => warnings.push(`${code}: ${message}`), + }); + + expect(found).toBeNull(); + expect(stored).toEqual({ localPath: built, providerUpload: null, providerName: null }); + expect(warnings).toEqual([ + 'provider-load: provider not usable (./cache.cjs): missing credentials; using local cache', + 'provider-load: provider not usable (./cache.cjs): missing credentials; using local cache', + ]); +}); + +test('a Metro-only provider adds no build tier', async () => { + const built = artifact('built'); + const local = capability({ resolve: () => null, store: () => built }); + const loadProvider = () => ({ name: './cache.cjs', provider: { metro: { get: () => null, set: () => {} } } }); + + expect(await resolveTieredBuild({ local: local.cap, loadProvider, target, destinationDir: workDir })).toBeNull(); + expect( + await storeTieredBuild({ local: local.cap, loadProvider, target, sourcePath: built, overwrite: false }), + ).toEqual({ localPath: built, providerUpload: null, providerName: null }); +}); + +test('the build budgets accept an environment override', () => { + expect(buildResolveTimeoutMs({})).toBe(BUILD_RESOLVE_TIMEOUT_MS); + expect(buildUploadTimeoutMs({})).toBe(BUILD_UPLOAD_TIMEOUT_MS); + expect(buildResolveTimeoutMs({ [BUILD_RESOLVE_TIMEOUT_ENV]: '1500' })).toBe(1500); + expect(buildUploadTimeoutMs({ [BUILD_UPLOAD_TIMEOUT_ENV]: '1500' })).toBe(1500); + expect(buildUploadTimeoutMs({ [BUILD_UPLOAD_TIMEOUT_ENV]: '-1' })).toBe(BUILD_UPLOAD_TIMEOUT_MS); +}); + +test('the destination is only prepared when a provider is about to be asked', async () => { + const prepared: string[] = []; + const localHit = capability({ resolve: () => '/cache/app.apk' }); + const localMiss = capability({ resolve: () => null }); + const downloaded = artifact('downloaded'); + const loadProvider = () => ({ + name: './cache.cjs', + provider: { builds: capability({ resolve: () => downloaded }).cap }, + }); + + await resolveTieredBuild({ + local: localHit.cap, + loadProvider, + target, + destinationDir: workDir, + ensureDestination: (dir) => prepared.push(dir), + }); + expect(prepared).toEqual([]); + + await resolveTieredBuild({ + local: localMiss.cap, + loadProvider, + target, + destinationDir: workDir, + ensureDestination: (dir) => prepared.push(dir), + skipRead: true, + }); + expect(prepared).toEqual([]); + + await resolveTieredBuild({ + local: localMiss.cap, + loadProvider, + target, + destinationDir: workDir, + ensureDestination: (dir) => prepared.push(dir), + }); + expect(prepared).toEqual([workDir]); +}); diff --git a/packages/cache/__tests__/contract.test.ts b/packages/cache/__tests__/contract.test.ts new file mode 100644 index 00000000..cf4ed130 --- /dev/null +++ b/packages/cache/__tests__/contract.test.ts @@ -0,0 +1,164 @@ +import { cpSync, mkdirSync, mkdtempSync, rmSync, existsSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { runCacheProviderContract } from '../contract.ts'; +import type { CacheProvider } from '../provider.ts'; + +let workDir: string; + +beforeEach(() => { + workDir = mkdtempSync(join(tmpdir(), 'stim-cache-contract-')); +}); + +afterEach(() => { + rmSync(workDir, { recursive: true, force: true }); +}); + +function memoryProvider(): CacheProvider { + const transforms = new Map(); + const builds = new Map(); + return { + metro: { + get: ({ key }) => transforms.get(key.toString('hex')) ?? null, + set: ({ key, value }) => { + transforms.set(key.toString('hex'), value); + }, + }, + builds: { + resolve: ({ key, destinationDir }) => { + const source = builds.get(key); + if (!source) return null; + const target = join(destinationDir, basename(source)); + cpSync(source, target, { recursive: true }); + return target; + }, + store: ({ key, sourcePath, overwrite }) => { + if (builds.has(key) && !overwrite) return; + const kept = join(workDir, `kept-${key}`); + mkdirSync(kept, { recursive: true }); + const target = join(kept, basename(sourcePath)); + cpSync(sourcePath, target, { recursive: true }); + builds.set(key, target); + }, + }, + }; +} + +test('the contract passes for a provider that honors both capabilities', async () => { + const results = await runCacheProviderContract({ + provider: memoryProvider(), + projectRoot: workDir, + workDir, + }); + + expect(results.length).toBe(12); + expect(results.filter((result) => !result.passed)).toEqual([]); + expect(new Set(results.map((result) => result.capability))).toEqual(new Set(['metro', 'builds'])); +}); + +test('the contract covers an iOS artifact directory', async () => { + const results = await runCacheProviderContract({ + provider: memoryProvider(), + projectRoot: workDir, + workDir, + platform: 'ios', + }); + + expect(results.filter((result) => !result.passed)).toEqual([]); +}); + +test('the contract only checks the capabilities a provider advertises', async () => { + const provider = memoryProvider(); + delete provider.builds; + + const results = await runCacheProviderContract({ provider, projectRoot: workDir, workDir }); + expect(new Set(results.map((result) => result.capability))).toEqual(new Set(['metro'])); +}); + +test('the contract reports violations instead of throwing', async () => { + const results = await runCacheProviderContract({ + provider: { + metro: { + get: () => Buffer.from('always the same'), + set: () => {}, + }, + builds: { + resolve: ({ destinationDir }) => join(destinationDir, 'missing.apk'), + store: () => {}, + }, + }, + projectRoot: workDir, + workDir, + }); + + const failed = results.filter((result) => !result.passed); + expect(failed.map((result) => result.name)).toEqual([ + 'metro get returns null or undefined for an unknown key', + 'metro set then get returns the stored buffer', + 'metro set then get returns the stored object', + 'metro keys do not collide', + 'builds resolve returns null for an unknown key', + 'builds store then resolve returns the same artifact', + 'builds store honors overwrite', + 'builds store keeps unrelated keys separate', + ]); + expect(failed[0]?.error).toMatch(/expected a miss/); + expect(existsSync(workDir)).toBe(true); +}); + +test('a capability that ignores its abort signal fails the contract', async () => { + const results = await runCacheProviderContract({ + provider: { + metro: { get: () => new Promise(() => {}), set: () => {} }, + }, + projectRoot: workDir, + workDir, + checkTimeoutMs: 400, + abortSettleMs: 100, + }); + + const aborting = results.find((result) => result.name === 'metro get settles when its signal aborts'); + expect(aborting?.passed).toBe(false); + expect(aborting?.error).toMatch(/did not settle within \d+ms/); +}, 30_000); + +test('the runner loads a module reference the way Stim does', async () => { + writeFileSync( + join(workDir, 'package.json'), + JSON.stringify({ name: 'contract-fixture', version: '1.0.0', type: 'module' }), + ); + writeFileSync( + join(workDir, 'cache.mjs'), + `export const apiVersion = 1; +export function createCacheProvider() { + const transforms = new Map(); + return { + metro: { + get: ({ key }) => transforms.get(key.toString('hex')) ?? null, + set: ({ key, value }) => { + transforms.set(key.toString('hex'), value); + }, + }, + }; +} +`, + ); + + const passing = await runCacheProviderContract({ + providerModule: './cache.mjs', + projectRoot: workDir, + workDir, + }); + expect(passing.filter((result) => !result.passed)).toEqual([]); + expect(new Set(passing.map((result) => result.capability))).toEqual(new Set(['metro'])); + + writeFileSync(join(workDir, 'old.mjs'), 'export const apiVersion = 2;\n'); + const rejected = await runCacheProviderContract({ + providerModule: './old.mjs', + projectRoot: workDir, + workDir, + }); + expect(rejected.length).toBe(1); + expect(rejected[0]).toMatchObject({ capability: 'module', passed: false }); + expect(rejected[0]?.error).toMatch(/apiVersion/); +}); diff --git a/packages/cache/__tests__/metro.test.ts b/packages/cache/__tests__/metro.test.ts new file mode 100644 index 00000000..84603294 --- /dev/null +++ b/packages/cache/__tests__/metro.test.ts @@ -0,0 +1,414 @@ +import { + METRO_READ_CONCURRENCY, + METRO_READ_FAILURE_LIMIT, + METRO_READ_TIMEOUT_ENV, + METRO_WRITE_TIMEOUT_ENV, + METRO_UPLOAD_CONCURRENCY, + METRO_UPLOAD_MAX_BYTES, + METRO_UPLOAD_MAX_ITEMS, + METRO_READ_TIMEOUT_MS, + METRO_WRITE_TIMEOUT_MS, + createTieredMetroStore, + metroCapabilityFromStore, + type MetroCacheStore, + type TieredMetroStoreOptions, +} from '../metro.ts'; +import type { LoadCacheProviderResult, MetroCacheCapability } from '../provider.ts'; + +function memoryStore(): MetroCacheStore & { entries: Map; cleared: number } { + const entries = new Map(); + return { + entries, + cleared: 0, + async get(key: Buffer) { + return entries.has(key.toString('hex')) ? entries.get(key.toString('hex')) : null; + }, + async set(key: Buffer, value: unknown) { + entries.set(key.toString('hex'), value); + }, + clear() { + this.cleared += 1; + }, + }; +} + +function tracked(capability: Partial = {}) { + const calls = { get: 0, set: 0 }; + const entries = new Map(); + const provider: MetroCacheCapability = { + get: async (input) => { + calls.get += 1; + if (capability.get) return capability.get(input); + return entries.get(input.key.toString('hex')) ?? null; + }, + set: async (input) => { + calls.set += 1; + if (capability.set) return capability.set(input); + entries.set(input.key.toString('hex'), input.value); + }, + }; + return { provider, calls, entries }; +} + +function store( + options: Partial & { + local: MetroCacheStore; + loadProvider: () => Promise; + }, +) { + const warnings: Array<{ code: string; message: string }> = []; + const tiered = createTieredMetroStore({ + projectRoot: '/repo/app', + cacheName: 'app', + warn: (code, message) => warnings.push({ code, message }), + ...options, + }); + return { tiered, warnings }; +} + +const KEY = Buffer.from('0123456789abcdef0123456789abcdef', 'hex'); + +test('the pinned queue and timeout limits stay put', () => { + expect(METRO_READ_TIMEOUT_MS).toBe(2_000); + expect(METRO_WRITE_TIMEOUT_MS).toBe(10_000); + expect(METRO_UPLOAD_CONCURRENCY).toBe(4); + expect(METRO_UPLOAD_MAX_ITEMS).toBe(128); + expect(METRO_UPLOAD_MAX_BYTES).toBe(32 * 1024 * 1024); +}); + +test('a local hit does not load or call the provider', async () => { + const local = memoryStore(); + await local.set(KEY, Buffer.from('cached')); + let loads = 0; + const { tiered } = store({ + local, + loadProvider: async () => { + loads += 1; + return { none: true }; + }, + }); + + expect(await tiered.get(KEY)).toEqual(Buffer.from('cached')); + expect(loads).toBe(0); +}); + +test('a provider hit is written locally before it is returned', async () => { + const local = memoryStore(); + const provider = tracked(); + await provider.provider.set({ + key: KEY, + value: Buffer.from('remote'), + projectRoot: '/repo/app', + cacheName: 'app', + signal: new AbortController().signal, + }); + const { tiered } = store({ + local, + loadProvider: async () => ({ name: './cache.cjs', provider: { metro: provider.provider } }), + }); + + expect(await tiered.get(KEY)).toEqual(Buffer.from('remote')); + expect(local.entries.get(KEY.toString('hex'))).toEqual(Buffer.from('remote')); +}); + +test('a total miss returns null and loads the provider once', async () => { + const local = memoryStore(); + const provider = tracked(); + let loads = 0; + const { tiered } = store({ + local, + loadProvider: async () => { + loads += 1; + return { name: './cache.cjs', provider: { metro: provider.provider } }; + }, + }); + + expect(await tiered.get(KEY)).toBeNull(); + expect(await tiered.get(Buffer.from('ff', 'hex'))).toBeNull(); + expect(loads).toBe(1); + expect(provider.calls.get).toBe(2); +}); + +test('set waits for the local write but not the provider write', async () => { + const local = memoryStore(); + let release = (): void => {}; + const blocked = new Promise((resolve) => { + release = () => resolve(); + }); + const provider = tracked({ set: async () => blocked }); + const { tiered } = store({ local, loadProvider: async () => ({ provider: { metro: provider.provider } }) }); + + await tiered.set(KEY, Buffer.from('fresh')); + expect(local.entries.get(KEY.toString('hex'))).toEqual(Buffer.from('fresh')); + expect(provider.calls.set).toBe(1); + + release(); + await tiered.flush(); +}); + +test('clear calls only the local store', async () => { + const local = memoryStore(); + const provider = tracked(); + const { tiered } = store({ local, loadProvider: async () => ({ provider: { metro: provider.provider } }) }); + + tiered.clear(); + expect(local.cleared).toBe(1); + expect(provider.calls.get + provider.calls.set).toBe(0); +}); + +test('a provider timeout returns a local miss and warns once', async () => { + const local = memoryStore(); + const provider = tracked({ get: () => new Promise(() => {}) }); + const { tiered, warnings } = store({ + local, + loadProvider: async () => ({ provider: { metro: provider.provider } }), + timeouts: { readMs: 5 }, + }); + + expect(await tiered.get(KEY)).toBeNull(); + expect(await tiered.get(KEY)).toBeNull(); + expect(warnings.map((w) => w.code)).toEqual(['provider-read']); + expect(warnings[0]?.message).toMatch(/within 5ms/); +}); + +test('one warning is emitted per failure class', async () => { + const local = memoryStore(); + const provider = tracked({ + get: () => { + throw new Error('read denied'); + }, + set: () => { + throw new Error('write denied'); + }, + }); + const { tiered, warnings } = store({ local, loadProvider: async () => ({ provider: { metro: provider.provider } }) }); + + await tiered.get(KEY); + await tiered.get(KEY); + await tiered.set(KEY, Buffer.from('a')); + await tiered.set(KEY, Buffer.from('b')); + await tiered.flush(); + + expect(warnings.map((w) => w.code)).toEqual(['provider-read', 'provider-write']); + expect(warnings[0]?.message).toMatch(/read denied/); + expect(warnings[1]?.message).toMatch(/write denied/); +}); + +test('an unusable provider warns once and keeps the local tier', async () => { + const local = memoryStore(); + const { tiered, warnings } = store({ + local, + loadProvider: async () => ({ name: './cache.cjs', unavailable: 'missing credentials' }), + }); + + expect(await tiered.get(KEY)).toBeNull(); + await tiered.set(KEY, Buffer.from('fresh')); + expect(local.entries.get(KEY.toString('hex'))).toEqual(Buffer.from('fresh')); + expect(warnings).toEqual([ + { + code: 'provider-load', + message: 'cache provider ./cache.cjs is not usable: missing credentials; using local transforms', + }, + ]); +}); + +test('the queue enforces item and byte limits', async () => { + const local = memoryStore(); + let release = (): void => {}; + const blocked = new Promise((resolve) => { + release = () => resolve(); + }); + const provider = tracked({ set: async () => blocked }); + const { tiered, warnings } = store({ + local, + loadProvider: async () => ({ provider: { metro: provider.provider } }), + limits: { concurrency: 1, maxItems: 2, maxBytes: 1024 }, + }); + + await tiered.set(Buffer.from('01', 'hex'), Buffer.alloc(8)); + await tiered.set(Buffer.from('02', 'hex'), Buffer.alloc(8)); + await tiered.set(Buffer.from('03', 'hex'), Buffer.alloc(8)); + expect(warnings.map((w) => w.code)).toEqual(['provider-queue']); + expect(provider.calls.set).toBe(1); + + release(); + await tiered.flush(); + expect(provider.calls.set).toBe(2); + + await tiered.set(Buffer.from('04', 'hex'), Buffer.alloc(2048)); + expect(provider.calls.set).toBe(2); + expect(local.entries.size).toBe(4); +}); + +test('a value that cannot be measured stays local', async () => { + const local = memoryStore(); + const provider = tracked(); + const { tiered, warnings } = store({ local, loadProvider: async () => ({ provider: { metro: provider.provider } }) }); + + const cyclic: Record = {}; + cyclic.self = cyclic; + await tiered.set(KEY, cyclic); + await tiered.flush(); + + expect(local.entries.get(KEY.toString('hex'))).toBe(cyclic); + expect(provider.calls.set).toBe(0); + expect(warnings.map((w) => w.code)).toEqual(['provider-write']); +}); + +test('a Metro store adapts to the capability contract', async () => { + const local = memoryStore(); + const capability = metroCapabilityFromStore(local); + await capability.set({ + key: KEY, + value: Buffer.from('x'), + projectRoot: '/repo', + cacheName: 'app', + signal: new AbortController().signal, + }); + expect( + await capability.get({ key: KEY, projectRoot: '/repo', cacheName: 'app', signal: new AbortController().signal }), + ).toEqual(Buffer.from('x')); +}); + +test('provider reads are capped so a slow provider cannot fan out per transform', async () => { + const local = memoryStore(); + let release = (): void => {}; + const blocked = new Promise((resolve) => { + release = () => resolve(); + }); + const provider = tracked({ get: async () => blocked.then(() => null) }); + const { tiered, warnings } = store({ + local, + loadProvider: async () => ({ provider: { metro: provider.provider } }), + limits: { readConcurrency: 2 }, + timeouts: { readMs: 5_000 }, + }); + + const reads = [ + tiered.get(Buffer.from('01', 'hex')), + tiered.get(Buffer.from('02', 'hex')), + tiered.get(Buffer.from('03', 'hex')), + tiered.get(Buffer.from('04', 'hex')), + ]; + expect(await Promise.all(reads.slice(2))).toEqual([null, null]); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(provider.calls.get).toBe(2); + expect(warnings.map((w) => w.code)).toEqual(['provider-read-busy']); + + release(); + expect(await Promise.all(reads.slice(0, 2))).toEqual([null, null]); +}); + +test('the provider tier switches off after repeated failures', async () => { + const local = memoryStore(); + const provider = tracked({ + get: () => { + throw new Error('unauthorized'); + }, + set: () => { + throw new Error('unauthorized'); + }, + }); + const { tiered, warnings } = store({ + local, + loadProvider: async () => ({ provider: { metro: provider.provider } }), + limits: { failureLimit: 2 }, + }); + + expect(await tiered.get(Buffer.from('01', 'hex'))).toBeNull(); + expect(await tiered.get(Buffer.from('02', 'hex'))).toBeNull(); + expect(await tiered.get(Buffer.from('03', 'hex'))).toBeNull(); + await tiered.set(Buffer.from('04', 'hex'), Buffer.from('fresh')); + await tiered.flush(); + + expect(provider.calls.get).toBe(2); + expect(provider.calls.set).toBe(0); + expect(warnings.map((w) => w.code)).toEqual(['provider-read', 'provider-disabled']); + expect(warnings[1]?.message).toMatch(/failed 2 times in a row/); + expect(local.entries.get('04')).toEqual(Buffer.from('fresh')); +}); + +test('a success resets the failure count', async () => { + const local = memoryStore(); + let calls = 0; + const provider = tracked({ + get: () => { + calls += 1; + if (calls === 2) return Buffer.from('hit'); + throw new Error('flaky'); + }, + }); + const { tiered, warnings } = store({ + local, + loadProvider: async () => ({ provider: { metro: provider.provider } }), + limits: { failureLimit: 2 }, + }); + + expect(await tiered.get(Buffer.from('01', 'hex'))).toBeNull(); + expect(await tiered.get(Buffer.from('02', 'hex'))).toEqual(Buffer.from('hit')); + expect(await tiered.get(Buffer.from('03', 'hex'))).toBeNull(); + expect(provider.calls.get).toBe(3); + expect(warnings.map((w) => w.code)).toEqual(['provider-read']); +}); + +test('an unusable provider disables the tier for writes too', async () => { + const local = memoryStore(); + let loads = 0; + const { tiered, warnings } = store({ + local, + loadProvider: async () => { + loads += 1; + return { name: './cache.cjs', unavailable: 'missing credentials' }; + }, + }); + + expect(await tiered.get(KEY)).toBeNull(); + await tiered.set(KEY, Buffer.from('fresh')); + await tiered.flush(); + expect(loads).toBe(1); + expect(warnings.map((w) => w.code)).toEqual(['provider-load']); +}); + +test('the read and write budgets accept an environment override', () => { + expect(METRO_READ_TIMEOUT_ENV).toBe('STIM_CACHE_METRO_READ_TIMEOUT_MS'); + expect(METRO_WRITE_TIMEOUT_ENV).toBe('STIM_CACHE_METRO_WRITE_TIMEOUT_MS'); + expect(METRO_READ_CONCURRENCY).toBe(6); + expect(METRO_READ_FAILURE_LIMIT).toBe(5); +}); + +test('a tripped breaker drops the queued backlog instead of draining it', async () => { + const local = memoryStore(); + let release = (): void => {}; + const blocked = new Promise((resolve) => { + release = () => resolve(); + }); + let writes = 0; + const provider = tracked({ + get: () => { + throw new Error('unauthorized'); + }, + set: async () => { + writes += 1; + return blocked; + }, + }); + const { tiered, warnings } = store({ + local, + loadProvider: async () => ({ provider: { metro: provider.provider } }), + limits: { concurrency: 1, failureLimit: 2 }, + }); + + await tiered.set(Buffer.from('01', 'hex'), Buffer.from('one')); + await tiered.set(Buffer.from('02', 'hex'), Buffer.from('two')); + await tiered.set(Buffer.from('03', 'hex'), Buffer.from('three')); + expect(writes).toBe(1); + + expect(await tiered.get(Buffer.from('04', 'hex'))).toBeNull(); + expect(await tiered.get(Buffer.from('05', 'hex'))).toBeNull(); + expect(warnings.map((w) => w.code)).toEqual(['provider-read', 'provider-disabled']); + + release(); + await tiered.flush(); + expect(writes).toBe(1); + expect(local.entries.size).toBe(3); +}); diff --git a/packages/cache/__tests__/provider.test.ts b/packages/cache/__tests__/provider.test.ts new file mode 100644 index 00000000..ce738fa2 --- /dev/null +++ b/packages/cache/__tests__/provider.test.ts @@ -0,0 +1,305 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + CACHE_PROVIDER_API_VERSION, + CACHE_PROVIDER_ENV, + CACHE_PROVIDER_ENV_NONE, + PROVIDER_LOAD_TIMEOUT_ENV, + PROVIDER_LOAD_TIMEOUT_MS, + cacheProviderEnvIsSet, + timeoutFromEnv, + cacheProviderConfigFromEnv, + cacheProviderEnv, + callWithTimeout, + createWarnOnce, + loadCacheProvider, + type CacheProviderConfig, +} from '../provider.ts'; + +let projectRoot: string; + +beforeEach(() => { + projectRoot = mkdtempSync(join(tmpdir(), 'stim-cache-provider-')); + writeFileSync(join(projectRoot, 'package.json'), JSON.stringify({ name: 'fixture', version: '1.0.0' })); +}); + +afterEach(() => { + rmSync(projectRoot, { recursive: true, force: true }); +}); + +function config(provider: string, options: Record = {}): CacheProviderConfig { + return { provider, options, baseDir: projectRoot }; +} + +function writeModule(file: string, source: string): void { + writeFileSync(join(projectRoot, file), source); +} + +test('an absent configuration reports no provider', async () => { + expect(await loadCacheProvider({ projectRoot, config: null })).toEqual({ none: true }); + expect(await loadCacheProvider({ projectRoot, config: config(' ') })).toEqual({ none: true }); +}); + +test('loads a CommonJS provider from the project root', async () => { + writeModule( + 'cache.cjs', + `exports.apiVersion = 1; +exports.createCacheProvider = ({ projectRoot, options }) => ({ + metro: { + get: () => null, + set: () => {}, + }, + builds: { + resolve: () => null, + store: () => {}, + }, + seen: { projectRoot, options }, +}); +`, + ); + + const loaded = await loadCacheProvider({ projectRoot, config: config('./cache.cjs', { bucket: 'mobile' }) }); + expect(loaded.name).toBe('./cache.cjs'); + expect(loaded.unavailable).toBeUndefined(); + expect(typeof loaded.provider?.metro?.get).toBe('function'); + expect(typeof loaded.provider?.builds?.resolve).toBe('function'); + expect((loaded.provider as unknown as { seen: unknown }).seen).toEqual({ + projectRoot, + options: { bucket: 'mobile' }, + }); +}); + +test('loads an ESM provider and passes the project root and options to the factory', async () => { + writeModule( + 'cache.mjs', + `import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export const apiVersion = 1; + +export function createCacheProvider({ projectRoot, options }) { + writeFileSync(join(projectRoot, 'factory.json'), JSON.stringify({ projectRoot, options })); + return { metro: { get: () => null, set: () => {} } }; +} +`, + ); + + const loaded = await loadCacheProvider({ projectRoot, config: config('./cache.mjs', { bucket: 'team' }) }); + expect(loaded.unavailable).toBeUndefined(); + expect(JSON.parse(readFileSync(join(projectRoot, 'factory.json'), 'utf-8'))).toEqual({ + projectRoot, + options: { bucket: 'team' }, + }); +}); + +test('unwraps a default export only when the namespace carries no apiVersion', async () => { + writeModule( + 'default.mjs', + `export default { + apiVersion: 1, + createCacheProvider: () => ({ builds: { resolve: () => null, store: () => {} } }), +}; +`, + ); + + const loaded = await loadCacheProvider({ projectRoot, config: config('./default.mjs') }); + expect(loaded.unavailable).toBeUndefined(); + expect(typeof loaded.provider?.builds?.store).toBe('function'); +}); + +test('resolves a package name from the base directory', async () => { + const packageDir = join(projectRoot, 'node_modules', 'team-cache'); + mkdirSync(packageDir, { recursive: true }); + writeFileSync(join(packageDir, 'package.json'), JSON.stringify({ name: 'team-cache', main: 'index.cjs' })); + writeFileSync( + join(packageDir, 'index.cjs'), + `exports.apiVersion = 1; +exports.createCacheProvider = () => ({ metro: { get: () => null, set: () => {} } }); +`, + ); + + const loaded = await loadCacheProvider({ projectRoot, config: config('team-cache') }); + expect(loaded.unavailable).toBeUndefined(); + expect(loaded.name).toBe('team-cache'); +}); + +test('accepts providers with only one capability', async () => { + writeModule( + 'metro-only.mjs', + `export const apiVersion = 1; +export const createCacheProvider = () => ({ metro: { get: () => null, set: () => {} } }); +`, + ); + writeModule( + 'builds-only.mjs', + `export const apiVersion = 1; +export const createCacheProvider = () => ({ builds: { resolve: () => null, store: () => {} } }); +`, + ); + + const metroOnly = await loadCacheProvider({ projectRoot, config: config('./metro-only.mjs') }); + expect(metroOnly.provider?.metro).toBeDefined(); + expect(metroOnly.provider?.builds).toBeUndefined(); + + const buildsOnly = await loadCacheProvider({ projectRoot, config: config('./builds-only.mjs') }); + expect(buildsOnly.provider?.builds).toBeDefined(); + expect(buildsOnly.provider?.metro).toBeUndefined(); +}); + +test('rejects an unsupported API version without throwing', async () => { + writeModule( + 'old.mjs', + `export const apiVersion = 2; +export const createCacheProvider = () => ({ metro: { get: () => null, set: () => {} } }); +`, + ); + + const loaded = await loadCacheProvider({ projectRoot, config: config('./old.mjs') }); + expect(loaded.provider).toBeUndefined(); + expect(loaded.unavailable).toMatch(/apiVersion/); + expect(CACHE_PROVIDER_API_VERSION).toBe(1); +}); + +test('rejects a module without a factory', async () => { + writeModule('no-factory.mjs', 'export const apiVersion = 1;\n'); + + const loaded = await loadCacheProvider({ projectRoot, config: config('./no-factory.mjs') }); + expect(loaded.unavailable).toMatch(/createCacheProvider/); +}); + +test('rejects malformed capability methods without throwing', async () => { + writeModule( + 'bad-metro.mjs', + `export const apiVersion = 1; +export const createCacheProvider = () => ({ metro: { get: () => null } }); +`, + ); + writeModule( + 'bad-builds.mjs', + `export const apiVersion = 1; +export const createCacheProvider = () => ({ builds: { store: () => {} } }); +`, + ); + writeModule( + 'empty.mjs', + `export const apiVersion = 1; +export const createCacheProvider = () => ({}); +`, + ); + + expect((await loadCacheProvider({ projectRoot, config: config('./bad-metro.mjs') })).unavailable).toMatch( + /metro.*get\(\) and set\(\)/, + ); + expect((await loadCacheProvider({ projectRoot, config: config('./bad-builds.mjs') })).unavailable).toMatch( + /builds.*resolve\(\) and store\(\)/, + ); + expect((await loadCacheProvider({ projectRoot, config: config('./empty.mjs') })).unavailable).toMatch( + /neither a metro nor a builds capability/, + ); +}); + +test('reports a missing module and a failing factory as unavailable', async () => { + const missing = await loadCacheProvider({ projectRoot, config: config('./nope.cjs') }); + expect(missing.provider).toBeUndefined(); + expect(missing.unavailable).toBeTruthy(); + + writeModule( + 'throws.mjs', + `export const apiVersion = 1; +export const createCacheProvider = () => { + throw new Error('missing credentials\\nsecond line'); +}; +`, + ); + const failed = await loadCacheProvider({ projectRoot, config: config('./throws.mjs') }); + expect(failed.unavailable).toBe('createCacheProvider() failed: missing credentials'); +}); + +test('the environment transport round-trips a configuration', () => { + const original: CacheProviderConfig = { + provider: './cache.cjs', + options: { bucket: 'mobile' }, + baseDir: projectRoot, + }; + const env = { [CACHE_PROVIDER_ENV]: cacheProviderEnv(original) }; + expect(cacheProviderConfigFromEnv(env)).toEqual(original); +}); + +test('the environment transport rejects malformed payloads', () => { + expect(cacheProviderConfigFromEnv({})).toBeNull(); + expect(cacheProviderConfigFromEnv({ [CACHE_PROVIDER_ENV]: 'not json' })).toBeNull(); + expect(cacheProviderConfigFromEnv({ [CACHE_PROVIDER_ENV]: '{"provider":"./x.cjs"}' })).toBeNull(); + expect(cacheProviderConfigFromEnv({ [CACHE_PROVIDER_ENV]: '{"baseDir":"/tmp"}' })).toBeNull(); + expect(cacheProviderConfigFromEnv({ [CACHE_PROVIDER_ENV]: '{"provider":"./x.cjs","baseDir":"/tmp"}' })).toEqual({ + provider: './x.cjs', + options: {}, + baseDir: '/tmp', + }); +}); + +test('a bounded call returns the value, the failure, or a timeout', async () => { + expect(await callWithTimeout(() => 'value', 1000)).toEqual({ value: 'value' }); + expect(await callWithTimeout(() => Promise.reject(new Error('boom\ndetail')), 1000)).toEqual({ failed: 'boom' }); + + let aborted = false; + const outcome = await callWithTimeout( + (signal) => + new Promise((resolve) => { + signal.addEventListener('abort', () => { + aborted = true; + resolve('late'); + }); + }), + 5, + ); + expect(outcome).toEqual({ timedOut: true }); + expect(aborted).toBe(true); +}); + +test('a warning class is emitted once', () => { + const lines: string[] = []; + const warn = createWarnOnce((line) => lines.push(line)); + warn('read', 'first read failure'); + warn('read', 'second read failure'); + warn('write', 'first write failure'); + expect(lines).toEqual(['first read failure', 'first write failure']); +}); + +test('a factory that never settles becomes unavailable instead of hanging', async () => { + writeModule( + 'hangs.mjs', + `export const apiVersion = 1; +export const createCacheProvider = () => new Promise(() => {}); +`, + ); + + const started = Date.now(); + const loaded = await loadCacheProvider({ projectRoot, config: config('./hangs.mjs'), timeoutMs: 25 }); + expect(loaded.provider).toBeUndefined(); + expect(loaded.unavailable).toBe('the module did not load within 25ms'); + expect(Date.now() - started).toBeLessThan(2_000); +}); + +test('a module whose top level never settles becomes unavailable instead of hanging', async () => { + writeModule('top-level-hang.mjs', 'await new Promise(() => {});\nexport const apiVersion = 1;\n'); + + const loaded = await loadCacheProvider({ projectRoot, config: config('./top-level-hang.mjs'), timeoutMs: 25 }); + expect(loaded.unavailable).toBe('the module did not load within 25ms'); +}); + +test('the load timeout is tunable through the environment', () => { + expect(timeoutFromEnv(PROVIDER_LOAD_TIMEOUT_ENV, PROVIDER_LOAD_TIMEOUT_MS, {})).toBe(PROVIDER_LOAD_TIMEOUT_MS); + expect(timeoutFromEnv(PROVIDER_LOAD_TIMEOUT_ENV, 10, { [PROVIDER_LOAD_TIMEOUT_ENV]: '250' })).toBe(250); + expect(timeoutFromEnv(PROVIDER_LOAD_TIMEOUT_ENV, 10, { [PROVIDER_LOAD_TIMEOUT_ENV]: '0' })).toBe(10); + expect(timeoutFromEnv(PROVIDER_LOAD_TIMEOUT_ENV, 10, { [PROVIDER_LOAD_TIMEOUT_ENV]: 'soon' })).toBe(10); +}); + +test('the environment transport carries an explicit none decision', () => { + expect(cacheProviderEnv(null)).toBe(CACHE_PROVIDER_ENV_NONE); + const env = { [CACHE_PROVIDER_ENV]: cacheProviderEnv(null) }; + expect(cacheProviderConfigFromEnv(env)).toBeNull(); + expect(cacheProviderEnvIsSet(env)).toBe(true); + expect(cacheProviderEnvIsSet({})).toBe(false); + expect(cacheProviderEnvIsSet({ [CACHE_PROVIDER_ENV]: ' ' })).toBe(false); +}); diff --git a/packages/cache/__tests__/timeout-process.test.ts b/packages/cache/__tests__/timeout-process.test.ts new file mode 100644 index 00000000..dc8c82a5 --- /dev/null +++ b/packages/cache/__tests__/timeout-process.test.ts @@ -0,0 +1,21 @@ +import { execFile } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const run = promisify(execFile); +const dist = join(fileURLToPath(import.meta.url), '..', '..', 'dist', 'index.mjs'); + +test('the timeout fires in a process with nothing else keeping the loop alive', async () => { + expect(existsSync(dist)).toBe(true); + + const script = ` + const { callWithTimeout } = await import(${JSON.stringify(dist)}); + const outcome = await callWithTimeout(() => new Promise(() => {}), 150); + process.stdout.write(JSON.stringify(outcome)); + `; + + const { stdout } = await run(process.execPath, ['--input-type=module', '-e', script], { timeout: 20_000 }); + expect(JSON.parse(stdout)).toEqual({ timedOut: true }); +}, 30_000); diff --git a/packages/cache/builds.ts b/packages/cache/builds.ts new file mode 100644 index 00000000..dac0f9c1 --- /dev/null +++ b/packages/cache/builds.ts @@ -0,0 +1,158 @@ +import { existsSync } from 'node:fs'; +import { + callWithTimeout, + timeoutFromEnv, + type BuildCacheCapability, + type BuildCacheTarget, + type LoadCacheProviderResult, + type ProviderCallResult, + type WarnOnce, +} from './provider.ts'; + +export const BUILD_RESOLVE_TIMEOUT_MS = 30_000; +export const BUILD_UPLOAD_TIMEOUT_MS = 60_000; + +export const BUILD_RESOLVE_TIMEOUT_ENV = 'STIM_CACHE_BUILD_RESOLVE_TIMEOUT_MS'; +export const BUILD_UPLOAD_TIMEOUT_ENV = 'STIM_CACHE_BUILD_UPLOAD_TIMEOUT_MS'; + +export function buildResolveTimeoutMs(env?: NodeJS.ProcessEnv): number { + return timeoutFromEnv(BUILD_RESOLVE_TIMEOUT_ENV, BUILD_RESOLVE_TIMEOUT_MS, env); +} + +export function buildUploadTimeoutMs(env?: NodeJS.ProcessEnv): number { + return timeoutFromEnv(BUILD_UPLOAD_TIMEOUT_ENV, BUILD_UPLOAD_TIMEOUT_MS, env); +} + +export interface TieredBuildResolution { + path: string; + tier: 'local' | 'provider'; + providerName?: string; + storedLocally?: boolean; +} + +export type LoadBuildProvider = () => Promise | LoadCacheProviderResult; + +export interface ResolveTieredBuildOptions { + local: BuildCacheCapability; + loadProvider?: LoadBuildProvider | null; + target: BuildCacheTarget; + destinationDir: string; + /** Called right before a provider is asked, so a local hit creates nothing on disk. */ + ensureDestination?: (destinationDir: string) => void; + skipRead?: boolean; + warn?: WarnOnce; + timeoutMs?: number; +} + +export interface StoreTieredBuildOptions { + local: BuildCacheCapability; + loadProvider?: LoadBuildProvider | null; + target: BuildCacheTarget; + sourcePath: string; + overwrite: boolean; + warn?: WarnOnce; + timeoutMs?: number; +} + +interface BuildProviderTier { + capability: BuildCacheCapability; + name: string; +} + +async function providerTier( + loadProvider: LoadBuildProvider | null | undefined, + warn: WarnOnce, +): Promise { + if (!loadProvider) return null; + const loaded = await loadProvider(); + if (loaded?.unavailable) { + warn( + 'provider-load', + `provider not usable (${loaded.name ?? 'the cache provider'}): ${loaded.unavailable}; using local cache`, + ); + return null; + } + const capability = loaded?.provider?.builds; + if (!capability) return null; + return { capability, name: loaded.name ?? 'the cache provider' }; +} + +export interface TieredBuildStoreResult { + localPath: string | null; + providerUpload: Promise> | null; + providerName: string | null; +} + +function neverAborted(): AbortSignal { + return new AbortController().signal; +} + +function ignore(): void {} + +export async function resolveTieredBuild({ + local, + loadProvider = null, + target, + destinationDir, + ensureDestination, + skipRead = false, + warn = ignore, + timeoutMs = buildResolveTimeoutMs(), +}: ResolveTieredBuildOptions): Promise { + if (skipRead) return null; + + const hit = await local.resolve({ ...target, destinationDir, signal: neverAborted() }); + if (hit) return { path: hit, tier: 'local' }; + + const tier = await providerTier(loadProvider, warn); + if (!tier) return null; + + const { capability: provider, name: label } = tier; + ensureDestination?.(destinationDir); + const outcome = await callWithTimeout((signal) => provider.resolve({ ...target, destinationDir, signal }), timeoutMs); + if (outcome.timedOut) { + warn('provider-resolve', `${label} did not answer within ${timeoutMs}ms; building instead`); + return null; + } + if (outcome.failed) { + warn('provider-resolve', `${label} could not be used: ${outcome.failed}; building instead`); + return null; + } + const path = typeof outcome.value === 'string' ? outcome.value.trim() : ''; + if (path === '') return null; + if (!existsSync(path)) { + warn('provider-resolve', `${label} returned ${path}, which does not exist; building instead`); + return null; + } + + let stored: string | null = null; + try { + stored = (await local.store({ ...target, sourcePath: path, overwrite: false, signal: neverAborted() })) || null; + } catch (error) { + warn( + 'provider-backfill', + `a ${label} hit could not be stored locally: ${String((error as Error)?.message || error)}`, + ); + } + return { path: stored || path, tier: 'provider', providerName: label, storedLocally: Boolean(stored) }; +} + +export async function storeTieredBuild({ + local, + loadProvider = null, + target, + sourcePath, + overwrite, + warn = ignore, + timeoutMs = buildUploadTimeoutMs(), +}: StoreTieredBuildOptions): Promise { + const localPath = (await local.store({ ...target, sourcePath, overwrite, signal: neverAborted() })) || null; + + const tier = await providerTier(loadProvider, warn); + if (!tier) return { localPath, providerUpload: null, providerName: null }; + + const providerUpload = callWithTimeout(async (signal) => { + await tier.capability.store({ ...target, sourcePath, overwrite, signal }); + }, timeoutMs); + return { localPath, providerUpload, providerName: tier.name }; +} diff --git a/packages/cache/contract.ts b/packages/cache/contract.ts new file mode 100644 index 00000000..2de8c78f --- /dev/null +++ b/packages/cache/contract.ts @@ -0,0 +1,407 @@ +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { join, relative, sep } from 'node:path'; +import { loadCacheProvider, type CacheProvider } from './provider.ts'; + +export interface CacheContractCheck { + name: string; + capability: 'metro' | 'builds'; + run(): Promise; +} + +export interface CacheContractResult { + name: string; + capability: 'metro' | 'builds' | 'module'; + passed: boolean; + error?: string; +} + +export interface CacheContractOptions { + provider: CacheProvider; + projectRoot: string; + workDir: string; + cacheName?: string; + platform?: 'ios' | 'android'; + checkTimeoutMs?: number; + abortSettleMs?: number; +} + +/** + * Loads the provider the way Stim does, from a module reference, so a provider + * author exercises `apiVersion` and factory validation as well as the + * capability checks. + */ +export interface CacheContractModuleOptions extends Omit { + providerModule: string; + options?: Record; + baseDir?: string; +} + +export const CACHE_CONTRACT_CHECK_TIMEOUT_MS = 30_000; + +const ABORT_SETTLE_MS = 1_000; + +async function settlesAfterAbort(start: (signal: AbortSignal) => unknown, settleMs: number): Promise { + const controller = new AbortController(); + const call = Promise.resolve() + .then(() => start(controller.signal)) + .then( + () => 'settled', + () => 'settled', + ); + controller.abort(); + let timer: NodeJS.Timeout | undefined; + const outcome = await Promise.race([ + call, + new Promise<'ignored'>((resolve) => { + timer = setTimeout(() => resolve('ignored'), settleMs); + }), + ]); + clearTimeout(timer); + assert(outcome === 'settled', `the call did not settle within ${settleMs}ms of its signal aborting`); +} + +function isInside(parent: string, child: string): boolean { + const rel = relative(parent, child); + return rel !== '' && !rel.startsWith(`..${sep}`) && rel !== '..'; +} + +function neverAborted(): AbortSignal { + return new AbortController().signal; +} + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function metroChecks({ + provider, + projectRoot, + cacheName = 'app', + abortSettleMs = ABORT_SETTLE_MS, +}: Required> & { + cacheName?: string; + abortSettleMs?: number; +}): CacheContractCheck[] { + const metro = provider.metro; + if (!metro) return []; + const context = { projectRoot, cacheName, signal: neverAborted() }; + + return [ + { + name: 'metro get returns null or undefined for an unknown key', + capability: 'metro', + async run() { + const value = await metro.get({ ...context, key: randomBytes(32) }); + assert(value === null || value === undefined, `expected a miss, received ${JSON.stringify(value)}`); + }, + }, + { + name: 'metro set then get returns the stored buffer', + capability: 'metro', + async run() { + const key = randomBytes(32); + const value = randomBytes(64); + await metro.set({ ...context, key, value }); + const stored = await metro.get({ ...context, key }); + assert(Buffer.isBuffer(stored), `expected a Buffer, received ${typeof stored}`); + assert(Buffer.compare(stored as Buffer, value) === 0, 'the stored buffer does not match the written buffer'); + }, + }, + { + name: 'metro set then get returns the stored object', + capability: 'metro', + async run() { + const key = randomBytes(32); + const value = { code: `contract-${randomUUID()}`, map: [1, 2, 3] }; + await metro.set({ ...context, key, value }); + const stored = await metro.get({ ...context, key }); + assert( + JSON.stringify(stored) === JSON.stringify(value), + `expected ${JSON.stringify(value)}, received ${JSON.stringify(stored)}`, + ); + }, + }, + { + name: 'metro get settles when its signal aborts', + capability: 'metro', + async run() { + await settlesAfterAbort( + (signal) => metro.get({ projectRoot, cacheName, key: randomBytes(32), signal }), + abortSettleMs, + ); + }, + }, + { + name: 'metro keys do not collide', + capability: 'metro', + async run() { + const first = randomBytes(32); + const second = randomBytes(32); + await metro.set({ ...context, key: first, value: Buffer.from('first') }); + await metro.set({ ...context, key: second, value: Buffer.from('second') }); + const stored = await metro.get({ ...context, key: first }); + assert( + Buffer.isBuffer(stored) && stored.toString() === 'first', + 'the second write overwrote the value of an unrelated key', + ); + }, + }, + ]; +} + +function buildChecks({ + provider, + projectRoot, + workDir, + platform = 'android', + abortSettleMs = ABORT_SETTLE_MS, +}: Required> & { + platform?: 'ios' | 'android'; + abortSettleMs?: number; +}): CacheContractCheck[] { + const builds = provider.builds; + if (!builds) return []; + const signal = neverAborted(); + + function artifact(): { sourcePath: string; contents: Buffer } { + const dir = join(workDir, `source-${randomUUID()}`); + mkdirSync(dir, { recursive: true }); + const sourcePath = join(dir, platform === 'ios' ? 'Contract.app' : 'contract.apk'); + const contents = randomBytes(128); + if (platform === 'ios') { + mkdirSync(sourcePath, { recursive: true }); + writeFileSync(join(sourcePath, 'contract.bin'), contents); + } else { + writeFileSync(sourcePath, contents); + } + return { sourcePath, contents }; + } + + function destination(): string { + const dir = join(workDir, `destination-${randomUUID()}`); + mkdirSync(dir, { recursive: true }); + return dir; + } + + function storedContents(path: string): Buffer { + return readFileSync(platform === 'ios' ? join(path, 'contract.bin') : path); + } + + return [ + { + name: 'builds resolve returns null for an unknown key', + capability: 'builds', + async run() { + const found = await builds.resolve({ + projectRoot, + platform, + key: `contract-miss-${randomUUID()}`, + destinationDir: destination(), + signal, + }); + assert(found === null || found === undefined, `expected a miss, received ${String(found)}`); + }, + }, + { + name: 'builds store then resolve returns the same artifact', + capability: 'builds', + async run() { + const key = `contract-${randomUUID()}`; + const { sourcePath, contents } = artifact(); + await builds.store({ projectRoot, platform, key, sourcePath, overwrite: false, signal }); + const found = await builds.resolve({ projectRoot, platform, key, destinationDir: destination(), signal }); + assert(typeof found === 'string' && found !== '', 'the stored key did not resolve'); + assert( + Buffer.compare(storedContents(found as string), contents) === 0, + 'the resolved artifact does not match the stored artifact', + ); + }, + }, + { + name: 'builds resolve settles when its signal aborts', + capability: 'builds', + async run() { + await settlesAfterAbort( + (aborting) => + builds.resolve({ + projectRoot, + platform, + key: `contract-abort-${randomUUID()}`, + destinationDir: destination(), + signal: aborting, + }), + abortSettleMs, + ); + }, + }, + { + name: 'builds resolve leaves the destination directory empty on a miss', + capability: 'builds', + async run() { + const dir = destination(); + await builds.resolve({ + projectRoot, + platform, + key: `contract-miss-${randomUUID()}`, + destinationDir: dir, + signal, + }); + assert(readdirSync(dir).length === 0, `a miss left ${readdirSync(dir).join(', ')} in the destination`); + }, + }, + { + name: 'builds resolve returns a path it owns or one inside the destination', + capability: 'builds', + async run() { + const key = `contract-${randomUUID()}`; + const { sourcePath, contents } = artifact(); + await builds.store({ projectRoot, platform, key, sourcePath, overwrite: false, signal }); + const dir = destination(); + const found = await builds.resolve({ projectRoot, platform, key, destinationDir: dir, signal }); + assert(typeof found === 'string' && found !== '', 'the stored key did not resolve'); + const path = found as string; + assert( + isInside(dir, path) || Buffer.compare(storedContents(path), contents) === 0, + `${path} is neither inside the destination nor a copy the capability owns`, + ); + }, + }, + { + name: 'builds store honors overwrite', + capability: 'builds', + async run() { + const key = `contract-${randomUUID()}`; + const first = artifact(); + const second = artifact(); + await builds.store({ projectRoot, platform, key, sourcePath: first.sourcePath, overwrite: false, signal }); + await builds.store({ projectRoot, platform, key, sourcePath: second.sourcePath, overwrite: false, signal }); + const kept = await builds.resolve({ projectRoot, platform, key, destinationDir: destination(), signal }); + assert(typeof kept === 'string', 'the stored key did not resolve'); + assert( + Buffer.compare(storedContents(kept as string), first.contents) === 0, + 'overwrite: false replaced an entry that already existed', + ); + + await builds.store({ projectRoot, platform, key, sourcePath: second.sourcePath, overwrite: true, signal }); + const replaced = await builds.resolve({ projectRoot, platform, key, destinationDir: destination(), signal }); + assert(typeof replaced === 'string', 'the overwritten key did not resolve'); + assert( + Buffer.compare(storedContents(replaced as string), second.contents) === 0, + 'overwrite: true kept the previous entry', + ); + }, + }, + { + name: 'builds store keeps unrelated keys separate', + capability: 'builds', + async run() { + const first = `contract-${randomUUID()}`; + const second = `contract-${randomUUID()}`; + const one = artifact(); + const two = artifact(); + await builds.store({ projectRoot, platform, key: first, sourcePath: one.sourcePath, overwrite: false, signal }); + await builds.store({ + projectRoot, + platform, + key: second, + sourcePath: two.sourcePath, + overwrite: false, + signal, + }); + const found = await builds.resolve({ + projectRoot, + platform, + key: first, + destinationDir: destination(), + signal, + }); + assert(typeof found === 'string' && found !== '', 'the first key stopped resolving after the second store'); + assert( + Buffer.compare(storedContents(found as string), one.contents) === 0, + 'the second store replaced the artifact of an unrelated key', + ); + }, + }, + ]; +} + +const MODULE_CHECK = 'the module loads through loadCacheProvider()'; + +async function loadContractProvider( + options: CacheContractModuleOptions, +): Promise { + const loaded = await loadCacheProvider({ + projectRoot: options.projectRoot, + config: { + provider: options.providerModule, + options: options.options ?? {}, + baseDir: options.baseDir ?? options.projectRoot, + }, + }); + if (!loaded.provider) { + return { + failure: { + name: MODULE_CHECK, + capability: 'module', + passed: false, + error: loaded.unavailable ?? 'the reference selected no provider', + }, + }; + } + return { ...options, provider: loaded.provider }; +} + +export function cacheProviderContractChecks(options: CacheContractOptions): CacheContractCheck[] { + return [ + ...metroChecks({ + provider: options.provider, + projectRoot: options.projectRoot, + ...(options.cacheName ? { cacheName: options.cacheName } : {}), + ...(options.abortSettleMs ? { abortSettleMs: options.abortSettleMs } : {}), + }), + ...buildChecks({ + provider: options.provider, + projectRoot: options.projectRoot, + workDir: options.workDir, + ...(options.platform ? { platform: options.platform } : {}), + ...(options.abortSettleMs ? { abortSettleMs: options.abortSettleMs } : {}), + }), + ]; +} + +export async function runCacheProviderContract( + options: CacheContractOptions | CacheContractModuleOptions, +): Promise { + const resolved = 'providerModule' in options ? await loadContractProvider(options) : options; + if ('failure' in resolved) return [resolved.failure]; + + const checkTimeoutMs = resolved.checkTimeoutMs ?? CACHE_CONTRACT_CHECK_TIMEOUT_MS; + const results: CacheContractResult[] = []; + for (const check of cacheProviderContractChecks(resolved)) { + try { + await withDeadline(check.run(), checkTimeoutMs); + results.push({ name: check.name, capability: check.capability, passed: true }); + } catch (error) { + results.push({ + name: check.name, + capability: check.capability, + passed: false, + error: String((error as Error)?.message || error), + }); + } + } + return results; +} + +async function withDeadline(work: Promise, timeoutMs: number): Promise { + let timer: NodeJS.Timeout | undefined; + const expired = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(`the check did not finish within ${timeoutMs}ms`)), timeoutMs); + }); + try { + await Promise.race([work, expired]); + } finally { + clearTimeout(timer); + } +} diff --git a/packages/cache/index.ts b/packages/cache/index.ts new file mode 100644 index 00000000..c92c22b2 --- /dev/null +++ b/packages/cache/index.ts @@ -0,0 +1,4 @@ +export * from './provider.ts'; +export * from './metro.ts'; +export * from './builds.ts'; +export * from './contract.ts'; diff --git a/packages/cache/metro.ts b/packages/cache/metro.ts new file mode 100644 index 00000000..610063d5 --- /dev/null +++ b/packages/cache/metro.ts @@ -0,0 +1,303 @@ +import { + callWithTimeout, + timeoutFromEnv, + type LoadCacheProviderResult, + type ProviderCallResult, + type MetroCacheCapability, + type WarnOnce, +} from './provider.ts'; + +export const METRO_READ_TIMEOUT_MS = 2_000; +export const METRO_WRITE_TIMEOUT_MS = 10_000; +export const METRO_READ_CONCURRENCY = 6; +export const METRO_READ_FAILURE_LIMIT = 5; +export const METRO_UPLOAD_CONCURRENCY = 4; +export const METRO_UPLOAD_MAX_ITEMS = 128; +export const METRO_UPLOAD_MAX_BYTES: number = 32 * 1024 * 1024; + +export const METRO_READ_TIMEOUT_ENV = 'STIM_CACHE_METRO_READ_TIMEOUT_MS'; +export const METRO_WRITE_TIMEOUT_ENV = 'STIM_CACHE_METRO_WRITE_TIMEOUT_MS'; + +/** + * Metro's structural cache-store contract: `get` returns the value or `null`, + * `set` stores it, and `clear` empties the store. + */ +export interface MetroCacheStore { + get(key: Buffer): unknown; + set(key: Buffer, value: unknown): unknown; + clear(): unknown; +} + +export interface TieredMetroStore extends MetroCacheStore { + get(key: Buffer): Promise; + set(key: Buffer, value: unknown): Promise; + clear(): unknown; + /** + * Resolves once queued provider writes have drained. Metro never calls it: + * every in-flight write already holds a referenced deadline, so the process + * drains on its own. Tests and embedders that own the process use it to wait + * without polling. + */ + flush(): Promise; +} + +export interface MetroTierLimits { + concurrency?: number; + maxItems?: number; + maxBytes?: number; + readConcurrency?: number; + failureLimit?: number; +} + +export interface MetroTierTimeouts { + readMs?: number; + writeMs?: number; +} + +export interface TieredMetroStoreOptions { + local: MetroCacheStore; + projectRoot: string; + cacheName: string; + loadProvider: () => Promise; + warn?: WarnOnce; + limits?: MetroTierLimits; + timeouts?: MetroTierTimeouts; +} + +export function metroCapabilityFromStore(store: MetroCacheStore): MetroCacheCapability { + return { + get: ({ key }) => store.get(key), + set: async ({ key, value }) => { + await store.set(key, value); + }, + }; +} + +const defaultWarn: WarnOnce = (_code, message) => { + process.stderr.write(`${message}\n`); +}; + +function onceByCode(warn: WarnOnce): WarnOnce { + const seen = new Set(); + return (code, message) => { + if (seen.has(code)) return; + seen.add(code); + warn(code, message); + }; +} + +function valueBytes(value: unknown): number | null { + if (Buffer.isBuffer(value)) return value.length; + try { + const json = JSON.stringify(value); + return json === undefined ? null : Buffer.byteLength(json); + } catch { + return null; + } +} + +interface UploadQueue { + add(bytes: number, run: () => Promise): boolean; + idle(): Promise; +} + +function createUploadQueue({ + concurrency, + maxItems, + maxBytes, +}: Required>): UploadQueue { + const pending: Array<{ bytes: number; run: () => Promise }> = []; + const waiters: Array<() => void> = []; + let active = 0; + let items = 0; + let bytes = 0; + + function settle(): void { + if (active > 0 || pending.length > 0) return; + while (waiters.length) waiters.shift()?.(); + } + + function pump(): void { + while (active < concurrency && pending.length > 0) { + const task = pending.shift()!; + active += 1; + void task + .run() + .catch(() => {}) + .finally(() => { + active -= 1; + items -= 1; + bytes -= task.bytes; + pump(); + settle(); + }); + } + } + + return { + add(size, run) { + if (items + 1 > maxItems || bytes + size > maxBytes) return false; + items += 1; + bytes += size; + pending.push({ bytes: size, run }); + pump(); + return true; + }, + idle() { + if (active === 0 && pending.length === 0) return Promise.resolve(); + return new Promise((resolve) => waiters.push(resolve)); + }, + }; +} + +export function createTieredMetroStore({ + local, + projectRoot, + cacheName, + loadProvider, + warn: emit = defaultWarn, + limits = {}, + timeouts = {}, +}: TieredMetroStoreOptions): TieredMetroStore { + const warn = onceByCode(emit); + const localCapability = metroCapabilityFromStore(local); + const readMs = timeouts.readMs ?? timeoutFromEnv(METRO_READ_TIMEOUT_ENV, METRO_READ_TIMEOUT_MS); + const writeMs = timeouts.writeMs ?? timeoutFromEnv(METRO_WRITE_TIMEOUT_ENV, METRO_WRITE_TIMEOUT_MS); + const readConcurrency = limits.readConcurrency ?? METRO_READ_CONCURRENCY; + const failureLimit = limits.failureLimit ?? METRO_READ_FAILURE_LIMIT; + const queue = createUploadQueue({ + concurrency: limits.concurrency ?? METRO_UPLOAD_CONCURRENCY, + maxItems: limits.maxItems ?? METRO_UPLOAD_MAX_ITEMS, + maxBytes: limits.maxBytes ?? METRO_UPLOAD_MAX_BYTES, + }); + + let loading: Promise | null = null; + let disabled = false; + let consecutiveFailures = 0; + let activeReads = 0; + + function recordFailure(): void { + consecutiveFailures += 1; + if (consecutiveFailures < failureLimit) return; + disabled = true; + warn( + 'provider-disabled', + `the cache provider failed ${consecutiveFailures} times in a row; this run keeps its transforms local`, + ); + } + + function providerCapability(): Promise { + if (disabled) return Promise.resolve(null); + loading ??= loadProvider().then((loaded) => { + if (loaded?.unavailable) { + disabled = true; + warn( + 'provider-load', + `cache provider ${loaded.name} is not usable: ${loaded.unavailable}; using local transforms`, + ); + return null; + } + return loaded?.provider?.metro ?? null; + }); + return loading; + } + + return { + async get(key) { + const hit = await localCapability.get({ key, projectRoot, cacheName, signal: neverAborted() }); + if (hit !== null && hit !== undefined) return hit; + + if (activeReads >= readConcurrency) { + warn( + 'provider-read-busy', + `the cache provider already has ${activeReads} reads in flight; further transforms read locally until it catches up`, + ); + return null; + } + + activeReads += 1; + let outcome: ProviderCallResult; + try { + const capability = await providerCapability(); + if (!capability) return null; + outcome = await callWithTimeout((signal) => capability.get({ key, projectRoot, cacheName, signal }), readMs); + } finally { + activeReads -= 1; + } + if (outcome.timedOut) { + recordFailure(); + warn( + 'provider-read', + `the cache provider did not answer a transform read within ${readMs}ms; using local transforms`, + ); + return null; + } + if (outcome.failed) { + recordFailure(); + warn( + 'provider-read', + `the cache provider could not read a transform: ${outcome.failed}; using local transforms`, + ); + return null; + } + consecutiveFailures = 0; + const value = outcome.value; + if (value === null || value === undefined) return null; + + try { + await localCapability.set({ key, value, projectRoot, cacheName, signal: neverAborted() }); + } catch (error) { + warn( + 'provider-backfill', + `a provider transform could not be written locally: ${String((error as Error)?.message || error)}`, + ); + } + return value; + }, + + async set(key, value) { + await localCapability.set({ key, value, projectRoot, cacheName, signal: neverAborted() }); + + const capability = await providerCapability(); + if (!capability) return; + + const bytes = valueBytes(value); + if (bytes === null) { + warn('provider-write', 'a transform could not be measured for the cache provider; it stays local'); + return; + } + const queued = queue.add(bytes, async () => { + if (disabled) return; + const outcome = await callWithTimeout( + (signal) => capability.set({ key, value, projectRoot, cacheName, signal }), + writeMs, + ); + if (outcome.timedOut || outcome.failed) recordFailure(); + else consecutiveFailures = 0; + if (outcome.timedOut) { + warn( + 'provider-write', + `the cache provider did not accept a transform within ${writeMs}ms; later transforms stay local until it answers`, + ); + } else if (outcome.failed) { + warn('provider-write', `the cache provider could not store a transform: ${outcome.failed}`); + } + }); + if (!queued) { + warn('provider-queue', 'the cache provider upload queue is full; those transforms stay local'); + } + }, + + clear() { + return local.clear(); + }, + + flush() { + return queue.idle(); + }, + }; +} + +function neverAborted(): AbortSignal { + return new AbortController().signal; +} diff --git a/packages/cache/package.json b/packages/cache/package.json new file mode 100644 index 00000000..a4dd12c7 --- /dev/null +++ b/packages/cache/package.json @@ -0,0 +1,32 @@ +{ + "name": "@stim-cli/cache", + "version": "1.0.0-rc.4", + "description": "Cache provider contract and tier coordination for Stim.", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/appandflow/stim.git", + "directory": "packages/cache" + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "type": "module", + "main": "dist/index.mjs", + "types": "dist/index.d.mts", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "module-sync": "./dist/index.mjs", + "default": "./dist/index.mjs" + } + }, + "scripts": { + "build": "tsdown" + }, + "engines": { + "node": "^20.19.4 || >=22.12.0" + } +} diff --git a/packages/cache/provider.ts b/packages/cache/provider.ts new file mode 100644 index 00000000..9ab42ced --- /dev/null +++ b/packages/cache/provider.ts @@ -0,0 +1,303 @@ +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +export const CACHE_PROVIDER_API_VERSION = 1 as const; + +export const CACHE_PROVIDER_ENV = 'STIM_CACHE_PROVIDER_CONFIG'; + +export const CACHE_PROVIDER_ENV_NONE = 'none'; + +export const PROVIDER_LOAD_TIMEOUT_MS = 10_000; + +export const PROVIDER_LOAD_TIMEOUT_ENV = 'STIM_CACHE_LOAD_TIMEOUT_MS'; + +/** + * Reads a positive integer millisecond override from the environment. Any other + * value falls back to the shipped default. + */ +export function timeoutFromEnv(name: string, fallback: number, env: NodeJS.ProcessEnv = process.env): number { + const raw = env[name]; + if (typeof raw !== 'string' || !/^[1-9]\d*$/.test(raw.trim())) return fallback; + const parsed = Number(raw.trim()); + return Number.isSafeInteger(parsed) ? parsed : fallback; +} + +/** + * One resolved provider selection: the module reference, the options passed to + * its factory, and the directory the reference resolves from. + */ +export interface CacheProviderConfig { + provider: string; + options: Record; + baseDir: string; +} + +export interface MetroCacheContext { + projectRoot: string; + cacheName: string; + signal: AbortSignal; +} + +export interface MetroCacheGetInput extends MetroCacheContext { + key: Buffer; +} + +export interface MetroCacheSetInput extends MetroCacheContext { + key: Buffer; + value: unknown; +} + +/** + * Metro transform cache. `get` returns `null` or `undefined` for a miss. `set` + * stores the value under the key. Both receive an `AbortSignal` the provider + * must honor. + */ +export interface MetroCacheCapability { + get(input: MetroCacheGetInput): unknown; + set(input: MetroCacheSetInput): void | Promise; +} + +export interface BuildCacheTarget { + projectRoot: string; + platform: 'ios' | 'android'; + key: string; +} + +export interface BuildCacheContext extends BuildCacheTarget { + signal: AbortSignal; +} + +export interface BuildResolveInput extends BuildCacheContext { + destinationDir: string; +} + +export interface BuildStoreInput extends BuildCacheContext { + sourcePath: string; + overwrite: boolean; +} + +/** + * Native build artifacts. + * + * `resolve` returns an existing path to the artifact for the key, or `null` for + * a miss. `destinationDir` is a scratch directory Stim creates and owns: a + * capability that fetches the artifact must materialize it there and return a + * path inside it, and a capability that already holds a local copy (the + * built-in filesystem tier) returns that copy instead. A miss must leave + * `destinationDir` empty. + * + * `store` publishes the `.app` directory or `.apk` file at `sourcePath`. + * `overwrite: false` must keep an entry that already exists for the key; + * `overwrite: true` must replace it. A capability that owns a local path + * returns it, and any other capability returns nothing. + * + * Stim owns fingerprints and cache keys; the capability owns transport, + * archive format, authentication, and retention. + */ +export interface BuildCacheCapability { + resolve(input: BuildResolveInput): string | null | Promise; + store(input: BuildStoreInput): string | null | void | Promise; +} + +export interface CacheProvider { + metro?: MetroCacheCapability; + builds?: BuildCacheCapability; +} + +/** + * The module shape a provider author exports. The loader rejects any other + * `apiVersion`. + */ +export interface CacheProviderModule { + apiVersion: typeof CACHE_PROVIDER_API_VERSION; + createCacheProvider(input: { + projectRoot: string; + options: Record; + }): CacheProvider | Promise; +} + +export interface LoadCacheProviderResult { + provider?: CacheProvider; + name?: string; + none?: true; + unavailable?: string; +} + +export interface ProviderCallResult { + value?: T; + failed?: string; + timedOut?: true; +} + +export type WarnOnce = (code: string, message: string) => void; + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function reason(error: unknown): string { + const message = String((error as Error)?.message || error || 'unknown error'); + return message.split('\n')[0]!.trim() || 'unknown error'; +} + +function moduleCandidate(namespace: unknown): unknown { + if (!isRecord(namespace)) return namespace; + if ('apiVersion' in namespace) return namespace; + return 'default' in namespace ? namespace.default : namespace; +} + +function capabilityError(provider: CacheProvider): string | null { + const metro = provider.metro; + if (metro !== undefined && (typeof metro?.get !== 'function' || typeof metro?.set !== 'function')) { + return 'the metro capability must implement get() and set()'; + } + const builds = provider.builds; + if (builds !== undefined && (typeof builds?.resolve !== 'function' || typeof builds?.store !== 'function')) { + return 'the builds capability must implement resolve() and store()'; + } + if (metro === undefined && builds === undefined) { + return 'the provider advertises neither a metro nor a builds capability'; + } + return null; +} + +export async function loadCacheProvider({ + projectRoot, + config, + timeoutMs = timeoutFromEnv(PROVIDER_LOAD_TIMEOUT_ENV, PROVIDER_LOAD_TIMEOUT_MS), +}: { + projectRoot: string; + config?: CacheProviderConfig | null; + timeoutMs?: number; +}): Promise { + const reference = typeof config?.provider === 'string' ? config.provider.trim() : ''; + if (!config || reference === '') return { none: true }; + + const outcome = await callWithTimeout(() => importCacheProvider({ projectRoot, config, reference }), timeoutMs); + if (outcome.timedOut) { + return { name: reference, unavailable: `the module did not load within ${timeoutMs}ms` }; + } + if (outcome.failed) return { name: reference, unavailable: outcome.failed }; + return outcome.value ?? { name: reference, unavailable: 'the module produced no provider' }; +} + +async function importCacheProvider({ + projectRoot, + config, + reference, +}: { + projectRoot: string; + config: CacheProviderConfig; + reference: string; +}): Promise { + const options = isRecord(config.options) ? config.options : {}; + const baseDir = typeof config.baseDir === 'string' && config.baseDir !== '' ? config.baseDir : projectRoot; + + let namespace: unknown; + try { + const resolved = createRequire(join(baseDir, 'package.json')).resolve(reference); + namespace = await import(pathToFileURL(resolved).href); + } catch (error) { + return { name: reference, unavailable: reason(error) }; + } + + const candidate = moduleCandidate(namespace); + if (!isRecord(candidate) || candidate.apiVersion !== CACHE_PROVIDER_API_VERSION) { + return { + name: reference, + unavailable: `expected apiVersion ${CACHE_PROVIDER_API_VERSION}, found ${JSON.stringify( + isRecord(candidate) ? candidate.apiVersion : candidate, + )}`, + }; + } + if (typeof candidate.createCacheProvider !== 'function') { + return { name: reference, unavailable: 'the module does not export createCacheProvider()' }; + } + + let provider: unknown; + try { + provider = await (candidate as unknown as CacheProviderModule).createCacheProvider({ projectRoot, options }); + } catch (error) { + return { name: reference, unavailable: `createCacheProvider() failed: ${reason(error)}` }; + } + if (!isRecord(provider)) { + return { name: reference, unavailable: 'createCacheProvider() did not return a provider object' }; + } + const invalid = capabilityError(provider as CacheProvider); + if (invalid) return { name: reference, unavailable: invalid }; + + return { name: reference, provider: provider as CacheProvider }; +} + +export async function callWithTimeout( + call: (signal: AbortSignal) => T | Promise, + timeoutMs: number, +): Promise> { + const controller = new AbortController(); + let timer: NodeJS.Timeout | null = null; + // The timer stays referenced on purpose: an unreferenced one lets Node exit + // while a provider call is pending, so the deadline never fires and the + // caller never gets its result. + const expired = new Promise<'timeout'>((resolve) => { + timer = setTimeout(() => { + controller.abort(); + resolve('timeout'); + }, timeoutMs); + }); + try { + const outcome = await Promise.race([ + Promise.resolve() + .then(() => call(controller.signal)) + .then( + (value) => ({ value }) as ProviderCallResult, + (error: unknown) => ({ failed: reason(error) }) as ProviderCallResult, + ), + expired, + ]); + return outcome === 'timeout' ? { timedOut: true } : outcome; + } finally { + if (timer) clearTimeout(timer); + } +} + +export function createWarnOnce(emit: (message: string) => void): WarnOnce { + const seen = new Set(); + return (code, message) => { + if (seen.has(code)) return; + seen.add(code); + emit(message); + }; +} + +export function cacheProviderEnv(config: CacheProviderConfig | null): string { + if (!config) return CACHE_PROVIDER_ENV_NONE; + return JSON.stringify({ provider: config.provider, options: config.options ?? {}, baseDir: config.baseDir }); +} + +/** + * True when a parent process decided the provider selection, including the + * `none` sentinel. A child must not run its own search once this is set. + */ +export function cacheProviderEnvIsSet(env: NodeJS.ProcessEnv = process.env): boolean { + const raw = env[CACHE_PROVIDER_ENV]; + return typeof raw === 'string' && raw.trim() !== ''; +} + +export function cacheProviderConfigFromEnv(env: NodeJS.ProcessEnv = process.env): CacheProviderConfig | null { + const raw = env[CACHE_PROVIDER_ENV]; + if (typeof raw !== 'string' || raw.trim() === '' || raw.trim() === CACHE_PROVIDER_ENV_NONE) return null; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + if (!isRecord(parsed)) return null; + const provider = parsed.provider; + const baseDir = parsed.baseDir; + if (typeof provider !== 'string' || provider.trim() === '' || typeof baseDir !== 'string' || baseDir === '') { + return null; + } + return { provider, options: isRecord(parsed.options) ? parsed.options : {}, baseDir }; +} diff --git a/packages/cache/tsconfig.json b/packages/cache/tsconfig.json new file mode 100644 index 00000000..f24de522 --- /dev/null +++ b/packages/cache/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "preserve", + "moduleResolution": "bundler", + "types": ["node", "vitest/globals"] + }, + "include": ["*.ts", "__tests__/**/*.ts"] +} diff --git a/packages/cache/tsdown.config.mts b/packages/cache/tsdown.config.mts new file mode 100644 index 00000000..d8dfb9dd --- /dev/null +++ b/packages/cache/tsdown.config.mts @@ -0,0 +1,12 @@ +import { defineConfig } from 'tsdown'; + +export default defineConfig({ + entry: { index: 'index.ts' }, + format: 'esm', + dts: true, + outDir: 'dist', + target: 'node20.19', + platform: 'node', + tsconfig: 'tsconfig.json', + fixedExtension: true, +}); diff --git a/packages/metro/README.md b/packages/metro/README.md index 20778a08..8d569bbf 100644 --- a/packages/metro/README.md +++ b/packages/metro/README.md @@ -19,6 +19,27 @@ config.cacheStores = sharedCacheStores('my-app'); Set `STIM_METRO_CACHE` to override the shared cache location. +## Optional second tier + +The filesystem cache above is always the first tier. When the project selects a +cache provider, the exported store reads it after a local miss, writes provider +hits back to the filesystem, and queues new transforms for the provider without +blocking Metro. Provider failures are misses. + +```json +{ + "cache": { + "provider": "./tools/cache-provider.cjs", + "options": { "bucket": "mobile-cache" } + } +} +``` + +Under `stim start` the supervisor passes the resolved selection to Metro. A +Metro process outside Stim reads the nearest committed `.stim.json`. See +[`@stim-cli/cache`](https://www.npmjs.com/package/@stim-cli/cache) for the +provider contract. `clear()` only clears the local tier. + ## Log reporter ```js diff --git a/packages/metro/__tests__/shared-cache-stores.test.ts b/packages/metro/__tests__/shared-cache-stores.test.ts new file mode 100644 index 00000000..3b6d27a3 --- /dev/null +++ b/packages/metro/__tests__/shared-cache-stores.test.ts @@ -0,0 +1,236 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + CACHE_PROVIDER_ENV, + cacheProviderEnv, + metroCapabilityFromStore, + runCacheProviderContract, + type CacheProviderConfig, + type LoadCacheProviderResult, + type MetroCacheStore, +} from '@stim-cli/cache'; +import { sharedStoreRoot } from '@stim-cli/core'; +import { sharedCacheStores } from '../index.ts'; + +const require = createRequire(import.meta.url); +const { FileStore } = require('metro-cache') as { FileStore: new (options: { root: string }) => MetroCacheStore }; + +let home: string; +let cacheDir: string; +let projectRoot: string; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'stim-metro-home-')); + cacheDir = mkdtempSync(join(tmpdir(), 'stim-metro-cache-')); + projectRoot = mkdtempSync(join(tmpdir(), 'stim-metro-project-')); + process.env.STIM_HOME = home; + process.env.STIM_METRO_CACHE = cacheDir; +}); + +afterEach(() => { + madeStores.length = 0; + rmSync(home, { recursive: true, force: true }); + rmSync(cacheDir, { recursive: true, force: true }); + rmSync(projectRoot, { recursive: true, force: true }); + delete process.env.STIM_HOME; + delete process.env.STIM_METRO_CACHE; +}); + +const madeStores: FakeStore[] = []; + +class FakeStore { + root: string; + entries = new Map(); + cleared = 0; + + constructor(options: { root: string }) { + this.root = options.root; + madeStores.push(this); + } + + get(key: Buffer): unknown { + return this.entries.get(key.toString('hex')) ?? null; + } + + set(key: Buffer, value: unknown): void { + this.entries.set(key.toString('hex'), value); + } + + clear(): void { + this.cleared += 1; + } +} + +const KEY = Buffer.from('a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4', 'hex'); + +test('without a configured provider the store is the plain file store', () => { + const stores = sharedCacheStores('demo', { FileStore: FakeStore, cwd: projectRoot, env: {} }); + + expect(stores.length).toBe(1); + expect(stores[0]).toBeInstanceOf(FakeStore); + expect((stores[0] as FakeStore).root).toBe(join(cacheDir, 'demo')); + expect(sharedStoreRoot(stores[0])).toBe(join(cacheDir, 'demo')); +}); + +test('the supervisor environment adds one tiered store on the same root', async () => { + const config: CacheProviderConfig = { provider: './cache.cjs', options: { bucket: 'mobile' }, baseDir: projectRoot }; + const seen: Array<{ projectRoot: string; config: CacheProviderConfig }> = []; + const remote = new Map([[KEY.toString('hex'), Buffer.from('from the provider')]]); + + const stores = sharedCacheStores('demo', { + FileStore: FakeStore, + cwd: projectRoot, + env: { [CACHE_PROVIDER_ENV]: cacheProviderEnv(config) }, + loadProvider: async (input): Promise => { + seen.push(input); + return { + name: input.config.provider, + provider: { + metro: { + get: ({ key }) => remote.get(key.toString('hex')) ?? null, + set: ({ key, value }) => { + remote.set(key.toString('hex'), value); + }, + }, + }, + }; + }, + }); + + const tiered = stores[0] as unknown as MetroCacheStore & { flush(): Promise }; + expect(tiered).not.toBeInstanceOf(FakeStore); + expect(sharedStoreRoot(tiered)).toBe(join(cacheDir, 'demo')); + expect(await tiered.get(KEY)).toEqual(Buffer.from('from the provider')); + expect(seen).toEqual([{ projectRoot, config }]); + + await tiered.set(Buffer.from('ff'.repeat(16), 'hex'), Buffer.from('fresh')); + await tiered.flush(); + expect(remote.get('ff'.repeat(16))).toEqual(Buffer.from('fresh')); +}); + +test('Metro running outside Stim reads the nearest committed provider', async () => { + const app = join(projectRoot, 'apps', 'mobile'); + mkdirSync(app, { recursive: true }); + mkdirSync(join(projectRoot, '.git'), { recursive: true }); + writeFileSync( + join(projectRoot, '.stim.json'), + JSON.stringify({ cache: { provider: './tools/cache.cjs', options: { bucket: 'team' } } }), + ); + const seen: Array<{ projectRoot: string; config: CacheProviderConfig }> = []; + + const stores = sharedCacheStores('demo', { + FileStore: FakeStore, + cwd: app, + env: {}, + loadProvider: async (input) => { + seen.push(input); + return { none: true }; + }, + }); + + await (stores[0] as unknown as MetroCacheStore).get(KEY); + expect(seen).toEqual([ + { + projectRoot: app, + config: { provider: './tools/cache.cjs', options: { bucket: 'team' }, baseDir: projectRoot }, + }, + ]); +}); + +test('clear only clears the local tier', async () => { + let providerCalls = 0; + const stores = sharedCacheStores('demo', { + FileStore: FakeStore, + cwd: projectRoot, + env: { + [CACHE_PROVIDER_ENV]: cacheProviderEnv({ provider: './cache.cjs', options: {}, baseDir: projectRoot }), + }, + loadProvider: async () => { + providerCalls += 1; + return { none: true }; + }, + }); + + (stores[0] as unknown as MetroCacheStore).clear(); + expect(madeStores.length).toBe(1); + expect(madeStores[0]?.cleared).toBe(1); + expect(providerCalls).toBe(0); +}); + +test('the built-in filesystem store satisfies the provider contract', async () => { + const results = await runCacheProviderContract({ + provider: { metro: metroCapabilityFromStore(new FileStore({ root: join(cacheDir, 'contract') })) }, + projectRoot, + workDir: projectRoot, + }); + + expect(results.length).toBeGreaterThan(0); + expect(results.filter((result) => !result.passed)).toEqual([]); +}); + +test('the committed search stops at the repository root', async () => { + const repo = join(projectRoot, 'repo'); + const app = join(repo, 'apps', 'mobile'); + mkdirSync(app, { recursive: true }); + mkdirSync(join(repo, '.git'), { recursive: true }); + writeFileSync(join(projectRoot, '.stim.json'), JSON.stringify({ cache: { provider: './outside-the-repo.cjs' } })); + const seen: unknown[] = []; + + const stores = sharedCacheStores('demo', { + FileStore: FakeStore, + cwd: app, + env: {}, + loadProvider: async (input) => { + seen.push(input); + return { none: true }; + }, + }); + + await (stores[0] as unknown as MetroCacheStore).get(KEY); + expect(seen).toEqual([]); + expect(stores[0]).toBeInstanceOf(FakeStore); +}); + +test('outside a repository only the starting directory is read', async () => { + const app = join(projectRoot, 'apps', 'mobile'); + mkdirSync(app, { recursive: true }); + writeFileSync(join(projectRoot, '.stim.json'), JSON.stringify({ cache: { provider: './parent.cjs' } })); + + expect(sharedCacheStores('demo', { FileStore: FakeStore, cwd: app, env: {} })[0]).toBeInstanceOf(FakeStore); + + writeFileSync(join(app, '.stim.json'), JSON.stringify({ cache: { provider: './here.cjs' } })); + const seen: Array<{ config: CacheProviderConfig }> = []; + const stores = sharedCacheStores('demo', { + FileStore: FakeStore, + cwd: app, + env: {}, + loadProvider: async (input) => { + seen.push(input); + return { none: true }; + }, + }); + await (stores[0] as unknown as MetroCacheStore).get(KEY); + expect(seen[0]?.config).toEqual({ provider: './here.cjs', options: {}, baseDir: app }); +}); + +test('an explicit none from the supervisor stops the committed search', async () => { + mkdirSync(join(projectRoot, '.git'), { recursive: true }); + writeFileSync(join(projectRoot, '.stim.json'), JSON.stringify({ cache: { provider: './committed.cjs' } })); + const seen: unknown[] = []; + + const stores = sharedCacheStores('demo', { + FileStore: FakeStore, + cwd: projectRoot, + env: { [CACHE_PROVIDER_ENV]: cacheProviderEnv(null) }, + loadProvider: async (input) => { + seen.push(input); + return { none: true }; + }, + }); + + expect(stores[0]).toBeInstanceOf(FakeStore); + await (stores[0] as unknown as MetroCacheStore).get(KEY); + expect(seen).toEqual([]); +}); diff --git a/packages/metro/index.ts b/packages/metro/index.ts index a8fc9fc4..ba2a48ce 100644 --- a/packages/metro/index.ts +++ b/packages/metro/index.ts @@ -1,6 +1,16 @@ import fs from 'node:fs'; import { createRequire } from 'node:module'; import path from 'node:path'; +import { + cacheProviderConfigFromEnv, + cacheProviderEnvIsSet, + createTieredMetroStore, + loadCacheProvider, + type CacheProviderConfig, + type LoadCacheProviderResult, + type MetroCacheStore, + type WarnOnce, +} from '@stim-cli/cache'; import { metroCacheRoot, METRO_NAMED_CACHE_LAYOUT, @@ -11,6 +21,14 @@ import { type FileStoreCtor = new (options: { root: string }) => object; +export interface SharedCacheStoresOptions { + FileStore?: FileStoreCtor; + env?: NodeJS.ProcessEnv; + cwd?: string; + loadProvider?: (input: { projectRoot: string; config: CacheProviderConfig }) => Promise; + warn?: WarnOnce; +} + const requireFromHere = createRequire(import.meta.url); // A Metro reporter event. Metro's event union is large and version-dependent, so @@ -68,7 +86,60 @@ function registerOnce( }); } -export function sharedCacheStores(name = 'app', { FileStore }: { FileStore?: FileStoreCtor } = {}): object[] { +function isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function repositoryRoot(startDir: string): string | null { + let dir = startDir; + for (;;) { + if (fs.existsSync(path.join(dir, '.git'))) return dir; + const parent = path.dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +function committedProviderConfig(startDir: string): CacheProviderConfig | null { + const start = path.resolve(startDir); + const stop = repositoryRoot(start) ?? start; + let dir = start; + for (;;) { + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(path.join(dir, '.stim.json'), 'utf-8')); + } catch { + parsed = null; + } + const cache = isPlainObject(parsed) && isPlainObject(parsed.cache) ? parsed.cache : null; + const reference = cache?.provider; + if (typeof reference === 'string' && reference.trim() !== '') { + return { + provider: reference.trim(), + options: isPlainObject(cache?.options) ? cache.options : {}, + baseDir: dir, + }; + } + const parent = path.dirname(dir); + if (dir === stop || parent === dir) return null; + dir = parent; + } +} + +function warnToStderr(_code: string, message: string): void { + process.stderr.write(`warning: ${message}\n`); +} + +export function sharedCacheStores( + name = 'app', + { + FileStore, + env = process.env, + cwd = process.cwd(), + loadProvider = loadCacheProvider, + warn = warnToStderr, + }: SharedCacheStoresOptions = {}, +): object[] { // metro-cache is a peer dependency that resolves at call time. const Store: FileStoreCtor = FileStore || (requireFromHere('metro-cache') as { FileStore: FileStoreCtor }).FileStore; const parent = cacheRoot(); @@ -87,7 +158,18 @@ export function sharedCacheStores(name = 'app', { FileStore }: { FileStore?: Fil }, ], ); - return [tagSharedStore(new Store({ root }), root)]; + const local = tagSharedStore(new Store({ root }), root); + const config = cacheProviderConfigFromEnv(env) ?? (cacheProviderEnvIsSet(env) ? null : committedProviderConfig(cwd)); + if (!config) return [local]; + + const tiered = createTieredMetroStore({ + local: local as MetroCacheStore, + projectRoot: path.resolve(cwd), + cacheName: name, + loadProvider: () => loadProvider({ projectRoot: path.resolve(cwd), config }), + warn, + }); + return [tagSharedStore(tiered, root)]; } const NDJSON_LEVELS = new Set(['debug', 'info', 'warn', 'error', 'fatal']); diff --git a/packages/metro/package.json b/packages/metro/package.json index 9bf1c7fe..0e1cb408 100644 --- a/packages/metro/package.json +++ b/packages/metro/package.json @@ -28,6 +28,7 @@ "build": "tsdown" }, "dependencies": { + "@stim-cli/cache": "^1.0.0-rc.4", "@stim-cli/core": "^1.0.0-rc.4" }, "peerDependencies": { diff --git a/packages/stim-cli/package.json b/packages/stim-cli/package.json index 18f7d2f4..3d097a32 100644 --- a/packages/stim-cli/package.json +++ b/packages/stim-cli/package.json @@ -46,6 +46,7 @@ }, "dependencies": { "@expo/fingerprint": "^0.20.10", + "@stim-cli/cache": "^1.0.0-rc.4", "@stim-cli/core": "^1.0.0-rc.4", "@stim-cli/metro": "^1.0.0-rc.4", "chalk": "^5.4.1", diff --git a/packages/stim-cli/src/__tests__/android-command.test.ts b/packages/stim-cli/src/__tests__/android-command.test.ts index f33c45f1..9caad09d 100644 --- a/packages/stim-cli/src/__tests__/android-command.test.ts +++ b/packages/stim-cli/src/__tests__/android-command.test.ts @@ -3180,3 +3180,194 @@ test('nothing recorded means nothing to wait for: the collector starts immediate }); expect(order).toEqual(['spawn']); }); + +describe('the project cache provider', () => { + const providerConfig = () => ({ provider: './cache.cjs', options: { bucket: 'mobile' }, baseDir: root }); + + function downloadedApk() { + const dir = join(root, 'provider-download'); + mkdirSync(dir, { recursive: true }); + const path = join(dir, 'app-debug.apk'); + writeFileSync(path, 'binary'); + return path; + } + + function providerOptions(builds: Record, extra: Record = {}) { + return { + resolveCacheProvider: () => providerConfig(), + loadCacheProviderModule: async () => ({ name: './cache.cjs', provider: { builds } }), + ...extra, + }; + } + + test('no configured provider never loads one', async () => { + let loads = 0; + const h = harness({ + loadCacheProviderModule: async () => { + loads += 1; + return { none: true }; + }, + }); + + expect((await h.run()).ok).toBe(true); + expect(loads).toBe(0); + expect(h.stderr.join('\n')).not.toMatch(/provider/); + }); + + test('a local hit does not load either second tier', async () => { + let loads = 0; + const h = harness({ + resolveCached: () => join(home, 'build-cache', 'android', CACHE_KEY, 'app-debug.apk'), + build: never('the build'), + storeCached: never('storeBuild'), + resolveCacheProvider: () => providerConfig(), + loadCacheProviderModule: async () => { + loads += 1; + return { name: './cache.cjs', provider: { builds: { resolve: () => null, store: () => {} } } }; + }, + }); + + expect((await h.run()).ok).toBe(true); + expect(loads).toBe(0); + expect(h.calls.loadProvider.length).toBe(0); + }); + + test('local, project provider, Expo provider, build lock, build is the order', async () => { + const timeline: string[] = []; + const h = harness( + providerOptions( + { + resolve: () => { + timeline.push('project provider'); + return null; + }, + store: () => {}, + }, + { + resolveCached: () => { + timeline.push('local'); + return null; + }, + loadProvider: async () => { + timeline.push('expo provider'); + return { provider: { plugin: {}, options: {} }, name: 'eas' }; + }, + acquireLock: () => { + timeline.push('build lock'); + return { + acquired: true as const, + path: join(home, 'build-locks', 'android-k.lock'), + lock: { + pid: process.pid, + projectRoot: root, + startedAt: new Date().toISOString(), + logFile: join(home, 'build-locks', 'android-k.log'), + }, + }; + }, + }, + ), + ); + + const result = await h.run(); + expect(result.ok).toBe(true); + expect(timeline).toEqual(['local', 'project provider', 'expo provider', 'build lock']); + expect(h.calls.build.length).toBe(1); + }); + + test('a provider hit installs the locally stored artifact without building', async () => { + const artifact = downloadedApk(); + const h = harness( + providerOptions({ + resolve: (input: { platform: string; key: string }) => { + expect(input).toMatchObject({ platform: 'android', key: CACHE_KEY }); + return artifact; + }, + store: () => {}, + }), + ); + + const result = await h.run(); + expect(result.ok).toBe(true); + expect(result.facts?.cacheHit).toBe('remote'); + expect(h.calls.build.length).toBe(0); + expect(h.calls.loadProvider.length).toBe(0); + expect(h.calls.storeCached[0]?.[2]).toBe(artifact); + expect(labelled(h.stderr, 'cache')[0]).toMatch(/provider hit \(\.\/cache\.cjs\) -> stored locally/); + expect(h.stdout.join('\n')).toMatch(/cache hit from \.\/cache\.cjs/); + expect(h.stdout.join('\n')).not.toMatch(/from the remote cache/); + }); + + test('a bare React Native project uses the provider without reading Expo config', async () => { + const artifact = downloadedApk(); + const h = harness(providerOptions({ resolve: () => artifact, store: () => {} })); + + expect((await h.run()).ok).toBe(true); + expect(h.calls.loadProvider.length).toBe(0); + }); + + test('a fresh build uploads to the provider and reports it', async () => { + const uploads: unknown[] = []; + const h = harness( + providerOptions({ + resolve: () => null, + store: (input: unknown) => { + uploads.push(input); + }, + }), + ); + + expect((await h.run()).ok).toBe(true); + expect(uploads.length).toBe(1); + expect(uploads[0]).toMatchObject({ platform: 'android', key: CACHE_KEY, overwrite: false }); + expect(labelled(h.stderr, 'cache').some((line) => line.includes('uploaded (./cache.cjs)'))).toBe(true); + }); + + test('an unusable provider reports once and the build still succeeds', async () => { + const h = harness({ + resolveCacheProvider: () => providerConfig(), + loadCacheProviderModule: async () => ({ name: './cache.cjs', unavailable: 'missing credentials' }), + }); + + expect((await h.run()).ok).toBe(true); + const notices = h.stderr.filter((line) => line.includes('provider not usable')); + expect(notices.length).toBe(1); + expect(notices[0]).toMatch(/provider not usable \(\.\/cache\.cjs\): missing credentials; using local cache/); + }); + + test('provider read and upload failures keep the build successful', async () => { + const h = harness( + providerOptions({ + resolve: () => { + throw new Error('unauthorized'); + }, + store: () => { + throw new Error('upload denied'); + }, + }), + ); + + expect((await h.run()).ok).toBe(true); + expect(h.stderr.join('\n')).toMatch(/\.\/cache\.cjs could not be used: unauthorized; building instead/); + expect(h.stderr.join('\n')).toMatch(/\.\/cache\.cjs upload failed: upload denied/); + }); + + test('--no-build-cache skips the provider read and still uploads', async () => { + const uploads: unknown[] = []; + const h = harness( + providerOptions( + { + resolve: never('the provider read'), + store: (input: unknown) => { + uploads.push(input); + }, + }, + { useBuildCache: false }, + ), + ); + + expect((await h.run()).ok).toBe(true); + expect(uploads.length).toBe(1); + expect(uploads[0]).toMatchObject({ overwrite: true }); + }); +}); diff --git a/packages/stim-cli/src/__tests__/build-cache.test.ts b/packages/stim-cli/src/__tests__/build-cache.test.ts index 43686fca..d9ebc18f 100644 --- a/packages/stim-cli/src/__tests__/build-cache.test.ts +++ b/packages/stim-cli/src/__tests__/build-cache.test.ts @@ -1,13 +1,26 @@ -import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync, existsSync, statSync, utimesSync } from 'node:fs'; +import { + mkdtempSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, + rmSync, + existsSync, + statSync, + utimesSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import assert from 'node:assert'; import * as expoFingerprint from '@expo/fingerprint'; import type { FingerprintSource, Options as FingerprintOptions } from '@expo/fingerprint'; +import { resolveTieredBuild, runCacheProviderContract, storeTieredBuild } from '@stim-cli/cache'; +import { readManifest } from '../cache-manifest.ts'; import { setExecutor, resetExecutor } from '../exec.ts'; import { artifactIn, buildCacheKey, + filesystemBuildCapability, compareSourceLists, describeFingerprintMiss, diffFingerprintSources, @@ -15,6 +28,8 @@ import { fingerprintDiffRecord, fingerprintDiffSuffix, fingerprintProject, + prepareProviderDownloadDir, + providerDownloadPath, refingerprintAfterMutation, resolveBuild, storeBuild, @@ -565,3 +580,141 @@ describe('untracked native files on a first miss', () => { expect(line).not.toMatch(/android\/d/); }); }); + +function treeOf(dir: string): string[] { + const entries: string[] = []; + const walk = (current: string, prefix: string): void => { + for (const name of readdirSync(current).toSorted()) { + const path = join(current, name); + const relative = prefix ? `${prefix}/${name}` : name; + if (statSync(path).isDirectory()) { + entries.push(`${relative}/`); + walk(path, relative); + } else { + entries.push(`${relative} ${readFileSync(path, 'utf-8')}`); + } + } + }; + walk(dir, ''); + return entries; +} + +test('the filesystem capability reads and writes the same entries as the plain functions', async () => { + const built = join(root, 'build', 'MyApp.app'); + mkdirSync(join(root, 'build'), { recursive: true }); + writeFileSync(built, 'binary'); + const sources = [fpFile('ios/Podfile', 'h1')]; + const key = buildCacheKey('ios', 'abc', {}); + + const direct = join(root, 'direct'); + storeBuild('ios', key, built, { root: direct, sources }); + + const throughCapability = join(root, 'capability'); + const capability = filesystemBuildCapability({ root: throughCapability, sources }); + await capability.store({ + projectRoot: root, + platform: 'ios', + key, + sourcePath: built, + overwrite: false, + signal: new AbortController().signal, + }); + + expect(treeOf(throughCapability)).toEqual(treeOf(direct)); + expect( + await capability.resolve({ + projectRoot: root, + platform: 'ios', + key, + destinationDir: root, + signal: new AbortController().signal, + }), + ).toBe(join(entryDir('ios', key, throughCapability), 'MyApp.app')); + expect(storedSources('ios', key, throughCapability)).toEqual(sources); +}); + +test('the tiered coordinator leaves the same cache on disk as a direct store', async () => { + const built = join(root, 'build', 'MyApp.apk'); + mkdirSync(join(root, 'build'), { recursive: true }); + writeFileSync(built, 'binary'); + const sources = [fpFile('android/build.gradle', 'h2')]; + const manifest: AssetManifest = { version: ASSET_MANIFEST_VERSION, assets: [] }; + const key = buildCacheKey('android', 'def', { variant: 'debug' }); + + const direct = join(root, 'direct'); + storeBuild('android', key, built, { root: direct, sources, assetManifest: manifest }); + + const tiered = join(root, 'tiered'); + const stored = await storeTieredBuild({ + local: filesystemBuildCapability({ root: tiered, sources, assetManifest: manifest }), + target: { projectRoot: root, platform: 'android', key }, + sourcePath: built, + overwrite: false, + }); + + expect(stored.providerUpload).toBeNull(); + expect(stored.localPath).toBe(join(entryDir('android', key, tiered), 'MyApp.apk')); + expect(treeOf(tiered)).toEqual(treeOf(direct)); + + const found = await resolveTieredBuild({ + local: filesystemBuildCapability({ root: tiered }), + target: { projectRoot: root, platform: 'android', key }, + destinationDir: root, + }); + expect(found).toEqual({ path: resolveBuild('android', key, tiered), tier: 'local' }); +}); + +test('a provider hit lands in the local cache under the same key', async () => { + const downloaded = join(root, 'downloaded', 'MyApp.apk'); + mkdirSync(join(root, 'downloaded'), { recursive: true }); + writeFileSync(downloaded, 'binary'); + const cacheDir = join(root, 'cache'); + const key = buildCacheKey('android', 'ghi', { variant: 'debug' }); + + const found = await resolveTieredBuild({ + local: filesystemBuildCapability({ root: cacheDir }), + loadProvider: () => ({ name: './cache.cjs', provider: { builds: { resolve: () => downloaded, store: () => {} } } }), + target: { projectRoot: root, platform: 'android', key }, + destinationDir: join(root, 'downloaded'), + }); + + expect(found).toEqual({ + path: join(entryDir('android', key, cacheDir), 'MyApp.apk'), + tier: 'provider', + providerName: './cache.cjs', + storedLocally: true, + }); + expect(resolveBuild('android', key, cacheDir)).toBe(found?.path); +}); + +test('the built-in filesystem build cache satisfies the provider contract', async () => { + for (const platform of ['ios', 'android'] as const) { + const results = await runCacheProviderContract({ + provider: { builds: filesystemBuildCapability({ root: join(root, `contract-${platform}`) }) }, + projectRoot: root, + workDir: root, + platform, + }); + expect(results.length).toBeGreaterThan(0); + expect(results.filter((result) => !result.passed)).toEqual([]); + } +}); + +test('the provider download directory is emptied and registered only when it is prepared', () => { + const workspace = join(root, 'workspace'); + const dir = providerDownloadPath(workspace); + expect(dir).toBe(join(workspace, 'cache-provider')); + expect(existsSync(dir)).toBe(false); + + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'leftover.apk'), 'from an interrupted run'); + prepareProviderDownloadDir(dir); + + expect(existsSync(dir)).toBe(true); + expect(existsSync(join(dir, 'leftover.apk'))).toBe(false); + const registered = readManifest().caches.find((cache) => cache.dir === dir); + assert(registered); + expect(registered.name).toBe('Cache provider downloads'); + expect(registered.prune).toBe('entries'); + expect(registered.entriesDepth).toBe(1); +}); diff --git a/packages/stim-cli/src/__tests__/guide.test.ts b/packages/stim-cli/src/__tests__/guide.test.ts index 0beb5ca1..0498a9fa 100644 --- a/packages/stim-cli/src/__tests__/guide.test.ts +++ b/packages/stim-cli/src/__tests__/guide.test.ts @@ -475,3 +475,21 @@ test('the binary command surface remains intentional', () => { 'worktree', ]); }); + +test('the guide documents the project cache provider as the tier between local and Expo', () => { + const lifecycle = renderTopic('lifecycle'); + const settings = renderTopic('settings'); + assert(lifecycle); + assert(settings); + + expect(lifecycle).toMatch(/THE BUILD CACHE HAS THREE LEVELS/); + expect(lifecycle).toMatch(/2\. The project's own cache provider[\s\S]*bare React\s+Native/i); + expect(lifecycle).toMatch(/3\. On an EXPO project only[\s\S]*Consulted only when levels one and two miss/i); + expect(lifecycle).toMatch(/ONE note per\s+failure class/i); + expect(lifecycle).toMatch(/gc[\s\S]*no delete\s+operation/i); + expect(settings).toMatch(/cache\.provider[\s\S]*@stim-cli\/cache/); + expect(settings).toMatch(/cache\.options[\s\S]*Keep secrets/); + expect(settings).toMatch(/cache\.provider[\s\S]*EXECUTABLE CODE/); + expect(settings).toMatch(/sharedCacheStores\(\)[\s\S]*stays local-only/); + expect(lifecycle).toMatch(/--no-build-cache looks nothing up -- not the local cache, not either/); +}); diff --git a/packages/stim-cli/src/__tests__/ios-command.test.ts b/packages/stim-cli/src/__tests__/ios-command.test.ts index c74fadbc..b68a5572 100644 --- a/packages/stim-cli/src/__tests__/ios-command.test.ts +++ b/packages/stim-cli/src/__tests__/ios-command.test.ts @@ -5,7 +5,7 @@ import { join } from 'node:path'; import { Command } from 'commander'; import { upsertProject } from '../config.ts'; import { parseNdjsonText } from '../ndjson.ts'; -import { workspaceLogsDir, workspaceStateFile } from '../paths.ts'; +import { workspaceDir, workspaceLogsDir, workspaceStateFile } from '../paths.ts'; import type { WorkspaceState } from '../supervisor/run.ts'; import { readWorkspaceState, writeWorkspaceState } from '../supervisor/run.ts'; import { @@ -2804,3 +2804,253 @@ describe('single-flight takeover says the previous build failed', () => { expect(line).toMatch(/build\.ndjson/); }); }); + +describe('the project cache provider', () => { + function providerConfig() { + return { provider: './cache.cjs', options: { bucket: 'mobile' }, baseDir: root }; + } + + function downloaded(name = 'Fixture.app') { + const dir = join(root, 'provider-download'); + mkdirSync(dir, { recursive: true }); + const path = join(dir, name); + writeFileSync(path, 'binary'); + return path; + } + + function providerDeps( + builds: Record, + record: (name: string, value: unknown) => void = () => {}, + ): LooseDeps { + return { + resolveCacheProviderConfig: () => providerConfig(), + loadCacheProvider: async (input) => { + record('loadCacheProvider', input); + return { name: './cache.cjs', provider: { builds } }; + }, + } as LooseDeps; + } + + test('no configured provider never loads one and keeps the existing order', async () => { + reserve(); + let loads = 0; + const { exitCode, calls, errs } = await run( + {}, + { + loadCacheProvider: async () => { + loads += 1; + return { none: true }; + }, + }, + ); + + expect(exitCode).toBe(null); + expect(loads).toBe(0); + expect(calls.order.filter((c) => ['resolveBuild', 'storeBuild'].includes(c))).toEqual([ + 'resolveBuild', + 'storeBuild', + ]); + expect(errs.join('\n')).not.toMatch(/provider/); + }); + + test('a local hit does not load either second tier', async () => { + reserve(); + let loads = 0; + const { calls } = await run( + {}, + { + resolveBuild: () => '/cache/Fixture.app', + ...providerDeps({ + resolve: () => { + throw new Error('the provider must not be consulted after a local hit'); + }, + store: () => {}, + }), + loadCacheProvider: async () => { + loads += 1; + return { name: './cache.cjs', provider: { builds: { resolve: () => null, store: () => {} } } }; + }, + }, + ); + + expect(loads).toBe(0); + expect(calls.order.includes('loadProjectProvider')).toBe(false); + expect(calls.order.includes('buildIos')).toBe(false); + }); + + test('a provider hit is stored locally and never reaches the Expo provider', async () => { + reserve(); + const artifact = downloaded(); + const seen: Array<{ name: string; value: unknown }> = []; + const { exitCode, calls, errs, logs } = await run( + { json: true }, + providerDeps( + { + resolve: (input: { key: string; platform: string; destinationDir: string }) => { + seen.push({ name: 'resolve', value: input }); + return artifact; + }, + store: () => {}, + }, + (name, value) => seen.push({ name, value }), + ), + ); + + expect(exitCode).toBe(null); + expect(calls.order.includes('buildIos')).toBe(false); + expect(calls.order.includes('loadProjectProvider')).toBe(false); + expect(calls.args.storeBuild.path).toBe(artifact); + expect(errs.join('\n')).toMatch(/^ {2}cache {7}provider hit \(\.\/cache\.cjs\) -> stored locally$/m); + expect(parseFirst(logs).cacheHit).toBe('remote'); + expect(seen[0]).toEqual({ name: 'loadCacheProvider', value: { projectRoot: root, config: providerConfig() } }); + expect(seen[1]?.value).toMatchObject({ platform: 'ios', key: `${FINGERPRINT}-debug-sim` }); + }); + + test('the summary names the provider a hit came from', async () => { + reserve(); + const artifact = downloaded(); + const { logs } = await run({}, providerDeps({ resolve: () => artifact, store: () => {} })); + + expect(logs.join('\n')).toMatch(/OK: [^\n]*from \.\/cache\.cjs/); + expect(logs.join('\n')).toMatch(/^ {2}cache {7}from \.\/cache\.cjs$/m); + expect(logs.join('\n')).not.toMatch(/the remote cache/); + }); + + test('a provider miss falls through to the Expo provider, the build lock, then the build', async () => { + reserve(); + const { exitCode, calls } = await run( + {}, + { + detectIsExpo: () => true, + ...providerDeps({ resolve: () => null, store: () => {} }), + }, + ); + + expect(exitCode).toBe(null); + expect( + calls.order.filter((c) => + ['resolveBuild', 'loadProjectProvider', 'acquireBuildLock', 'buildIos', 'storeBuild'].includes(c), + ), + ).toEqual(['resolveBuild', 'loadProjectProvider', 'acquireBuildLock', 'buildIos', 'storeBuild']); + }); + + test('a fresh build uploads to the provider and the Expo provider independently', async () => { + reserve(); + const uploads: unknown[] = []; + const { exitCode, errs, calls } = await run( + {}, + { + detectIsExpo: () => true, + loadProjectProvider: async () => ({ provider: { plugin: {}, options: {} }, name: 'eas' }), + ...providerDeps({ + resolve: () => null, + store: (input: unknown) => { + uploads.push(input); + }, + }), + }, + ); + + expect(exitCode).toBe(null); + expect(calls.order.includes('uploadRemote')).toBe(true); + expect(uploads.length).toBe(1); + expect(uploads[0]).toMatchObject({ platform: 'ios', key: `${FINGERPRINT}-debug-sim`, overwrite: false }); + expect(errs.join('\n')).toMatch(/^ {2}cache {7}uploaded \(\.\/cache\.cjs\)$/m); + }); + + test('--no-build-cache skips the provider read and still uploads', async () => { + reserve(); + const uploads: unknown[] = []; + const { exitCode, calls } = await run( + { buildCache: false }, + providerDeps({ + resolve: () => { + throw new Error('the provider must not be read with --no-build-cache'); + }, + store: (input: unknown) => { + uploads.push(input); + }, + }), + ); + + expect(exitCode).toBe(null); + expect(calls.order.includes('buildIos')).toBe(true); + expect(uploads.length).toBe(1); + expect(uploads[0]).toMatchObject({ overwrite: true }); + }); + + test('an unusable provider reports once and the build still succeeds', async () => { + reserve(); + const { exitCode, errs, calls } = await run( + {}, + { + resolveCacheProviderConfig: () => providerConfig(), + loadCacheProvider: async () => ({ name: './cache.cjs', unavailable: 'missing credentials' }), + }, + ); + + expect(exitCode).toBe(null); + expect(calls.order.includes('buildIos')).toBe(true); + const notices = errs.filter((line) => /provider not usable/.test(line)); + expect(notices.length).toBe(1); + expect(notices[0]).toMatch(/provider not usable \(\.\/cache\.cjs\): missing credentials; using local cache/); + }); + + test('a provider read failure and an upload failure keep the build successful', async () => { + reserve(); + const { exitCode, errs, calls } = await run( + {}, + providerDeps({ + resolve: () => { + throw new Error('unauthorized'); + }, + store: () => { + throw new Error('upload denied'); + }, + }), + ); + + expect(exitCode).toBe(null); + expect(calls.order.includes('buildIos')).toBe(true); + expect(errs.join('\n')).toMatch(/\.\/cache\.cjs could not be used: unauthorized; building instead/); + expect(errs.join('\n')).toMatch(/\.\/cache\.cjs upload failed: upload denied/); + }); +}); + +test('an invalid cache.provider setting is reported once and the run continues', async () => { + reserve(); + writeFileSync(join(root, '.stim.json'), JSON.stringify({ cache: { provider: 42 } })); + const { exitCode, errs, calls } = await run( + {}, + { + repoRoot: () => root, + loadCacheProvider: async () => { + throw new Error('an invalid setting must not reach the loader'); + }, + }, + ); + + expect(exitCode).toBe(null); + expect(calls.order.includes('buildIos')).toBe(true); + const notices = errs.filter((line) => line.includes('Invalid cache.provider setting')); + expect(notices.length).toBe(1); + expect(notices[0]).toMatch(/Using the local cache\./); +}); + +test('a local hit leaves no provider download directory behind', async () => { + reserve(); + const { exitCode } = await run( + {}, + { + resolveBuild: () => '/cache/Fixture.app', + resolveCacheProviderConfig: () => ({ provider: './cache.cjs', options: {}, baseDir: root }), + loadCacheProvider: async () => ({ + name: './cache.cjs', + provider: { builds: { resolve: () => null, store: () => {} } }, + }), + }, + ); + + expect(exitCode).toBe(null); + expect(existsSync(join(workspaceDir(root), 'cache-provider'))).toBe(false); +}); diff --git a/packages/stim-cli/src/__tests__/settings.test.ts b/packages/stim-cli/src/__tests__/settings.test.ts index b6b908f8..4b7ee4e8 100644 --- a/packages/stim-cli/src/__tests__/settings.test.ts +++ b/packages/stim-cli/src/__tests__/settings.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'os'; import { join } from 'path'; import { androidAvdConfigSetting, + cacheProviderSettingError, androidAvdConfigSettingError, androidDataPartitionSizeBytes, androidDataPartitionSizeGbSetting, @@ -18,6 +19,7 @@ import { remoteAndroidSetting, remoteDeviceSettingError, remoteIosSetting, + resolveCacheProviderConfig, resolveSettings, tunnelModeSetting, unknownSettingKeys, @@ -394,3 +396,95 @@ describe('ngrokUrlSetting', () => { expect(ngrokUrlSetting({ metro: { tunnel: 'ngrok', ngrokUrl: 42 } })).toBeNull(); }); }); + +test('resolveCacheProviderConfig reports no provider when nothing configures one', () => { + writeFileSync(join(tmpHome, '.stim.json'), JSON.stringify({ worktree: { baseRef: 'fresh' } })); + upsertProject('/proj', {}); + + expect( + resolveCacheProviderConfig({ projectPath: '/proj', gitCommonDir: '/repo/.git', repoRoot: tmpHome }), + ).toBeNull(); +}); + +test('a committed provider resolves from the directory holding .stim.json', () => { + writeFileSync( + join(tmpHome, '.stim.json'), + JSON.stringify({ cache: { provider: './tools/cache-provider.cjs', options: { bucket: 'mobile' } } }), + ); + upsertProject('/proj', {}); + + expect(resolveCacheProviderConfig({ projectPath: '/proj', gitCommonDir: '/repo/.git', repoRoot: tmpHome })).toEqual({ + provider: './tools/cache-provider.cjs', + options: { bucket: 'mobile' }, + baseDir: tmpHome, + }); +}); + +test('machine project settings override repository and committed providers', () => { + writeFileSync(join(tmpHome, '.stim.json'), JSON.stringify({ cache: { provider: './committed.cjs' } })); + setRepoSetting('/repo/.git', 'cache', { provider: './repo.cjs' }); + upsertProject('/proj', {}); + setProjectSetting('/proj', 'cache', { provider: './project.cjs' }); + + expect(resolveCacheProviderConfig({ projectPath: '/proj', gitCommonDir: '/repo/.git', repoRoot: tmpHome })).toEqual({ + provider: './project.cjs', + options: {}, + baseDir: '/proj', + }); +}); + +test('machine repository settings override committed providers and resolve from the repository root', () => { + writeFileSync(join(tmpHome, '.stim.json'), JSON.stringify({ cache: { provider: './committed.cjs' } })); + setRepoSetting('/repo/.git', 'cache', { provider: './repo.cjs' }); + upsertProject('/proj', {}); + + expect(resolveCacheProviderConfig({ projectPath: '/proj', gitCommonDir: '/repo/.git', repoRoot: tmpHome })).toEqual({ + provider: './repo.cjs', + options: {}, + baseDir: tmpHome, + }); +}); + +test('provider options merge across layers with earlier layers winning', () => { + writeFileSync( + join(tmpHome, '.stim.json'), + JSON.stringify({ cache: { provider: './committed.cjs', options: { bucket: 'team', region: 'us' } } }), + ); + setRepoSetting('/repo/.git', 'cache', { options: { region: 'eu' } }); + upsertProject('/proj', {}); + setProjectSetting('/proj', 'cache', { options: { token: 'from-machine' } }); + + expect(resolveCacheProviderConfig({ projectPath: '/proj', gitCommonDir: '/repo/.git', repoRoot: tmpHome })).toEqual({ + provider: './committed.cjs', + options: { token: 'from-machine', region: 'eu', bucket: 'team' }, + baseDir: tmpHome, + }); +}); + +test('an invalid provider reference reports no provider and names the error', () => { + writeFileSync(join(tmpHome, '.stim.json'), JSON.stringify({ cache: { provider: 42, options: { a: 1 } } })); + upsertProject('/proj', {}); + + const context = { projectPath: '/proj', gitCommonDir: '/repo/.git', repoRoot: tmpHome }; + expect(resolveCacheProviderConfig(context)).toBeNull(); + expect(cacheProviderSettingError(resolveSettings(context))).toBe( + 'Invalid cache.provider setting 42. Expected a module path or package name.', + ); +}); + +test('cacheProviderSettingError accepts valid shapes and names invalid ones', () => { + expect(cacheProviderSettingError({})).toBeNull(); + expect(cacheProviderSettingError({ cache: { provider: './cache.cjs', options: { bucket: 'a' } } })).toBeNull(); + expect(cacheProviderSettingError({ cache: { provider: ' ' } })).toMatch(/Invalid cache\.provider setting/); + expect(cacheProviderSettingError({ cache: { provider: './cache.cjs', options: 'nope' } })).toMatch( + /Invalid cache\.options setting/, + ); + expect(cacheProviderSettingError({ cache: 'nope' })).toMatch(/Invalid cache setting/); +}); + +test('cache.provider and cache.options are known settings', () => { + expect( + unknownSettingKeys({ cache: { provider: './cache.cjs', options: { bucket: 'a', nested: { deep: true } } } }), + ).toEqual([]); + expect(unknownSettingKeys({ cache: { unknown: true } })).toEqual(['cache.unknown']); +}); diff --git a/packages/stim-cli/src/__tests__/start.test.ts b/packages/stim-cli/src/__tests__/start.test.ts index 8689553e..71af4547 100644 --- a/packages/stim-cli/src/__tests__/start.test.ts +++ b/packages/stim-cli/src/__tests__/start.test.ts @@ -5,6 +5,7 @@ import { createServer, type Server } from 'node:http'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; +import { CACHE_PROVIDER_ENV, CACHE_PROVIDER_ENV_NONE, cacheProviderConfigFromEnv } from '@stim-cli/cache'; import { getProject, upsertProject } from '../config.ts'; import { resetExecutor, setExecutor } from '../exec.ts'; import { @@ -727,6 +728,86 @@ describe('action: spawning the supervisor', () => { expect(facts.mode).toBe('bare-inproc'); }); + test('a configured cache provider reaches the supervisor through the environment', async () => { + const port = 8155; + writeFileSync( + join(root, '.stim.json'), + JSON.stringify({ cache: { provider: './tools/cache-provider.cjs', options: { bucket: 'mobile' } } }), + ); + const exec = metroExecutor({ listeners: {} }); + const held: { server: Server | null } = { server: null }; + exec.spawn = (cmd, args, opts) => { + exec.calls.spawn.push({ cmd, args, opts }); + writeWorkspaceState(root, { supervisor: { pid: process.pid, port, mode: 'bare-inproc', startedAt: 'T' } }); + metroListener(port).then((s) => { + held.server = s; + exec.listening = true; + return s; + }); + return { pid: process.pid, unref() {}, on() {} }; + }; + const base = exec.runQuiet.bind(exec); + exec.runQuiet = (cmd) => { + if (new RegExp(`lsof -nP -iTCP:${port}`).test(cmd)) return exec.listening ? '5150' : ''; + return base(cmd); + }; + setExecutor(exec); + upsertProject(root, { metroPort: port }); + + try { + await runAction({ json: true, wait: '10' }); + } finally { + held.server?.close(); + } + + const spawned = exec.calls.spawn[0]; + assert(spawned); + const env = spawned.opts.env as NodeJS.ProcessEnv; + expect(cacheProviderConfigFromEnv(env)).toEqual({ + provider: './tools/cache-provider.cjs', + options: { bucket: 'mobile' }, + baseDir: root, + }); + expect(process.env[CACHE_PROVIDER_ENV]).toBeUndefined(); + }); + + test('no configured provider hands the supervisor an explicit none', async () => { + const port = 8157; + process.env[CACHE_PROVIDER_ENV] = 'stale'; + const exec = metroExecutor({ listeners: {} }); + const held: { server: Server | null } = { server: null }; + exec.spawn = (cmd, args, opts) => { + exec.calls.spawn.push({ cmd, args, opts }); + writeWorkspaceState(root, { supervisor: { pid: process.pid, port, mode: 'bare-inproc', startedAt: 'T' } }); + metroListener(port).then((s) => { + held.server = s; + exec.listening = true; + return s; + }); + return { pid: process.pid, unref() {}, on() {} }; + }; + const base = exec.runQuiet.bind(exec); + exec.runQuiet = (cmd) => { + if (new RegExp(`lsof -nP -iTCP:${port}`).test(cmd)) return exec.listening ? '5150' : ''; + return base(cmd); + }; + setExecutor(exec); + upsertProject(root, { metroPort: port }); + + try { + await runAction({ json: true, wait: '10' }); + } finally { + held.server?.close(); + delete process.env[CACHE_PROVIDER_ENV]; + } + + const spawned = exec.calls.spawn[0]; + assert(spawned); + const env = spawned.opts.env as NodeJS.ProcessEnv; + expect(env[CACHE_PROVIDER_ENV]).toBe(CACHE_PROVIDER_ENV_NONE); + expect(cacheProviderConfigFromEnv(env)).toBeNull(); + }); + test('start --remote passes --tunnel to an Expo supervisor in explicit expo mode', async () => { const { exec } = await runSpawnedExpoStart({ port: 8156, diff --git a/packages/stim-cli/src/build-cache.ts b/packages/stim-cli/src/build-cache.ts index cfca1276..cc2515c1 100644 --- a/packages/stim-cli/src/build-cache.ts +++ b/packages/stim-cli/src/build-cache.ts @@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, u import { basename, dirname, join } from 'path'; import * as expoFingerprint from '@expo/fingerprint'; import type { Fingerprint, FingerprintSource, Options as FingerprintOptions } from '@expo/fingerprint'; +import { buildUploadTimeoutMs, type BuildCacheCapability, type ProviderCallResult } from '@stim-cli/cache'; import { buildCacheKey as coreBuildCacheKey } from '@stim-cli/core'; import { getExecutor } from './exec.ts'; import { register } from './cache-manifest.ts'; @@ -135,6 +136,75 @@ export function storeBuild( return artifactIn(dest); } +export const PROVIDER_DOWNLOAD_DIR = 'cache-provider'; + +export function providerDownloadPath(workspacePath: string): string { + return join(workspacePath, PROVIDER_DOWNLOAD_DIR); +} + +/** + * Empties and registers the scratch directory a provider downloads into. It is + * created only when a provider is about to be asked, and registered so `gc` + * reports what an interrupted run left behind. + */ +export function prepareProviderDownloadDir(dir: string): void { + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + try { + register({ + dir, + name: 'Cache provider downloads', + prune: 'entries', + entriesDepth: 1, + note: 'artifacts fetched from the project cache provider; anything left here is from an interrupted run', + }); + } catch {} +} + +export interface FilesystemBuildCapabilityOptions { + root?: string; + sources?: FingerprintSource[] | null; + assetManifest?: AssetManifest | null; + resolve?: typeof resolveBuild; + store?: typeof storeBuild; +} + +export function filesystemBuildCapability({ + root, + sources, + assetManifest, + resolve = resolveBuild, + store = storeBuild, +}: FilesystemBuildCapabilityOptions = {}): BuildCacheCapability { + const stored = { + ...(root === undefined ? {} : { root }), + ...(sources === undefined ? {} : { sources }), + ...(assetManifest === undefined ? {} : { assetManifest }), + }; + return { + resolve: ({ platform, key }) => (root === undefined ? resolve(platform, key) : resolve(platform, key, root)), + store: ({ platform, key, sourcePath, overwrite }) => store(platform, key, sourcePath, { ...stored, overwrite }), + }; +} + +export interface ProviderUploadOutcome { + line: string; + warn: boolean; +} + +export function providerUploadOutcome( + result: ProviderCallResult | null | undefined, + name: string | null, +): ProviderUploadOutcome | null { + if (!result) return null; + const label = name || 'the cache provider'; + if (result.timedOut) { + return { line: `${label} upload was cancelled after ${buildUploadTimeoutMs()}ms`, warn: true }; + } + if (result.failed) return { line: `${label} upload failed: ${result.failed}`, warn: true }; + return { line: `uploaded (${label})`, warn: false }; +} + const SOURCES_FILE = 'fingerprint-sources.json'; export function storedSources(platform: string, key: string, root: string = cacheRoot()): FingerprintSource[] | null { diff --git a/packages/stim-cli/src/commands/android.ts b/packages/stim-cli/src/commands/android.ts index c4a44523..11b787e7 100644 --- a/packages/stim-cli/src/commands/android.ts +++ b/packages/stim-cli/src/commands/android.ts @@ -4,6 +4,14 @@ import { existsSync, mkdirSync, openSync, readFileSync, readdirSync, rmSync } fr import { basename, join, relative } from 'node:path'; import { spawnEntry } from '../spawn-entry.ts'; import { InvalidArgumentError, type Command } from 'commander'; +import { + createWarnOnce, + loadCacheProvider, + resolveTieredBuild, + storeTieredBuild, + type LoadCacheProviderResult, + type ProviderCallResult, +} from '@stim-cli/cache'; import type { AndroidFacts, RemoteDeviceBackend, SettingsObject, WaitedForBuild } from '../types.ts'; import { formatDuration, phaseLine, shortHash } from '../command-output.ts'; import { getConcurrencyLimits, getProject, upsertProject } from '../config.ts'; @@ -11,9 +19,13 @@ import { getExecutor } from '../exec.ts'; import { buildCacheKey, describeFingerprintMiss, + filesystemBuildCapability, fingerprintDiffRecord, fingerprintDiffSuffix, fingerprintProject, + prepareProviderDownloadDir, + providerDownloadPath, + providerUploadOutcome, refingerprintAfterMutation, resolveBuild, storeBuild, @@ -33,7 +45,7 @@ import { import { acquireBuildSlot, releaseBuildSlot, type BuildSlotHandle } from '../engine/build-slots.ts'; import { createNdjsonWriter } from '../ndjson.ts'; import { isPidAlive, resolveProjectMetro } from '../metro.ts'; -import { emulatorLogFile, workspaceLogsDir } from '../paths.ts'; +import { emulatorLogFile, workspaceDir, workspaceLogsDir } from '../paths.ts'; import { detectAndroidPackage, detectBundleId, detectIsExpo, findProjectRoot, projectShortcut } from '../project.ts'; import { devClientScheme as configuredDevClientScheme, @@ -49,9 +61,11 @@ import { REMOTE_DEVICE_BACKENDS, androidAvdConfigSettingError, androidDataPartitionSizeGbSettingError, + cacheProviderSettingError, publicUrlSetting, remoteAndroidSetting, remoteDeviceSettingError, + resolveCacheProviderConfig, resolveSettings, tunnelModeSetting, } from '../settings.ts'; @@ -639,6 +653,8 @@ interface RunAndroidOptions { easAuth?: typeof checkEasAuth; resolveRemoteBuild?: typeof resolveRemote; uploadRemoteBuild?: typeof uploadRemote; + resolveCacheProvider?: typeof resolveCacheProviderConfig; + loadCacheProviderModule?: typeof loadCacheProvider; needsPrebuildFor?: typeof needsPrebuild; prebuild?: typeof runPrebuild; build?: typeof buildAndroid; @@ -844,6 +860,7 @@ interface ReportAndroidResultArgs { storeKey: string; waitedForBuild: WaitedForBuild | null; remote: LoadProjectProviderResult | null; + providerName: string | null; launchState: boolean | string; launched: LaunchResultLike; writer: AndroidWriter; @@ -866,6 +883,7 @@ function reportAndroidResult({ storeKey, waitedForBuild, remote, + providerName, launchState, launched, writer, @@ -898,7 +916,7 @@ function reportAndroidResult({ const summary = `OK: ${androidPackage} launched on ${serial}, ` + `${release ? `${variant} (embedded JS, no Metro)` : `Metro port ${metroPort}`} ` + - `(${cacheOutcome(record.cacheHit, remote?.name)})`; + `(${cacheOutcome(record.cacheHit, remote?.name ?? providerName)})`; const outcome = launchState === LAUNCH_UNVERIFIED ? chalk.yellow(`${summary} -- launch UNVERIFIED`) @@ -906,7 +924,7 @@ function reportAndroidResult({ ? chalk.green(`${summary} -- bundle requested, still building`) : chalk.green(summary); const deviceName = record.avdName || record.deviceName || serial; - const cacheResult = useBuildCache ? cacheOutcome(record.cacheHit, remote?.name) : 'bypassed; built'; + const cacheResult = useBuildCache ? cacheOutcome(record.cacheHit, remote?.name ?? providerName) : 'bypassed; built'; const metroResult = release ? `embedded (${variant})` : !metroCheck @@ -951,6 +969,8 @@ interface FinishAndroidRunArgs { storeKey: string; waitedForBuild: WaitedForBuild | null; uploadPending: Promise | null; + providerUpload: Promise> | null; + providerName: string | null; remote: LoadProjectProviderResult | null; abandonedRemote: boolean; started: number; @@ -1001,6 +1021,8 @@ async function finishAndroidRun({ storeKey, waitedForBuild, uploadPending, + providerUpload, + providerName, remote, abandonedRemote: remoteWasAbandoned, started, @@ -1139,6 +1161,8 @@ async function finishAndroidRun({ } const uploadWasAbandoned = await finishAndroidUpload(uploadPending, remote, phase); + const providerOutcome = providerUploadOutcome(providerUpload ? await providerUpload : null, providerName); + if (providerOutcome) phase('cache', providerOutcome.warn ? chalk.yellow(providerOutcome.line) : providerOutcome.line); persistLastBuild({ writeState, root, record, startedAt, durationMs: now() - started, status: 'ok', out }); @@ -1192,6 +1216,7 @@ async function finishAndroidRun({ storeKey, waitedForBuild, remote, + providerName, launchState, launched, writer, @@ -1242,6 +1267,8 @@ function resolveRunAndroidOptions( easAuth = checkEasAuth, resolveRemoteBuild = resolveRemote, uploadRemoteBuild = uploadRemote, + resolveCacheProvider = resolveCacheProviderConfig, + loadCacheProviderModule = loadCacheProvider, needsPrebuildFor = needsPrebuild, prebuild = runPrebuild, build = buildAndroid, @@ -1300,6 +1327,8 @@ function resolveRunAndroidOptions( easAuth, resolveRemoteBuild, uploadRemoteBuild, + resolveCacheProvider, + loadCacheProviderModule, needsPrebuildFor, prebuild, build, @@ -1360,6 +1389,8 @@ export async function runAndroid(options: RunAndroidOptions = {} as RunAndroidOp easAuth, resolveRemoteBuild, uploadRemoteBuild, + resolveCacheProvider, + loadCacheProviderModule, needsPrebuildFor, prebuild, build, @@ -1464,11 +1495,15 @@ export async function runAndroid(options: RunAndroidOptions = {} as RunAndroidOp const settingsRepoRoot = repoRoot(root); const settingsRoot = settingsRepoRoot ?? root; - const settings = resolveSettingsFor({ + const settingsContext = { projectPath: root, gitCommonDir: gitCommonDir(root), repoRoot: settingsRepoRoot, - }); + }; + const settings = resolveSettingsFor(settingsContext); + const cacheProviderConfig = resolveCacheProvider(settingsContext); + const cacheProviderError = cacheProviderSettingError(settings); + if (cacheProviderError) out(phaseLine('cache', chalk.yellow(`${cacheProviderError} Using the local cache.`))); const dataPartitionSizeError = androidDataPartitionSizeGbSettingError(settings); if (dataPartitionSizeError) { return fail( @@ -1676,6 +1711,13 @@ export async function runAndroid(options: RunAndroidOptions = {} as RunAndroidOp record.deviceName = device.deviceName ?? device.avdName ?? null; let hash = ''; + let providerUpload: Promise> | null = null; + let providerName: string | null = null; + let providerLoad: Promise | null = null; + const cacheWarn = createWarnOnce((line) => phase('cache', chalk.yellow(line))); + const loadTieredProvider = cacheProviderConfig + ? () => (providerLoad ??= loadCacheProviderModule({ projectRoot: root, config: cacheProviderConfig })) + : null; let fingerprintSources: FingerprintSource[] = []; let cacheKey = ''; let storeHash = ''; @@ -1712,7 +1754,16 @@ export async function runAndroid(options: RunAndroidOptions = {} as RunAndroidOp storeKey = cacheKey; storeSources = fingerprintSources; - const cached = useBuildCache ? resolveCached(PLATFORM, cacheKey) : null; + const found = await resolveTieredBuild({ + local: filesystemBuildCapability({ resolve: resolveCached, store: storeCached, sources: fingerprintSources }), + loadProvider: loadTieredProvider, + target: { projectRoot: root, platform: PLATFORM, key: cacheKey }, + destinationDir: providerDownloadPath(workspaceDir(root)), + ensureDestination: prepareProviderDownloadDir, + skipRead: !useBuildCache, + warn: cacheWarn, + }); + const cached = found?.tier === 'local' ? found.path : null; record.cacheHit = cached ? 'local' : false; record.cacheSkipped = !useBuildCache; let missDiff = ''; @@ -1736,7 +1787,12 @@ export async function runAndroid(options: RunAndroidOptions = {} as RunAndroidOp `${shortHash(hash)} ${cached ? 'hit' : 'miss'}${useBuildCache ? '' : ' (--no-build-cache)'} ${fingerprintTimer()}${missDiff}`, ); if (missUntracked) phase('fingerprint', chalk.dim(missUntracked)); - apkPath = cached || null; + if (found?.tier === 'provider') { + record.cacheHit = 'remote'; + providerName = found.providerName ?? null; + phase('cache', `provider hit (${providerName})${found.storedLocally ? ' -> stored locally' : ''}`); + } + apkPath = found?.path ?? null; return true; } @@ -2013,11 +2069,21 @@ export async function runAndroid(options: RunAndroidOptions = {} as RunAndroidOp const assetManifest = release ? captureAssets(root, { variant }) : null; try { - storeCached(PLATFORM, storeKey, apkPath!, { + const stored = await storeTieredBuild({ + local: filesystemBuildCapability({ + resolve: resolveCached, + store: storeCached, + sources: storeSources, + assetManifest, + }), + loadProvider: loadTieredProvider, + target: { projectRoot: root, platform: PLATFORM, key: storeKey }, + sourcePath: apkPath!, overwrite: !useBuildCache || swapFellBack, - sources: storeSources, - assetManifest, + warn: cacheWarn, }); + providerUpload = stored.providerUpload; + providerName = stored.providerName ?? providerName; } catch (err) { phase('cache', chalk.yellow(`could not store the build: ${(err as Error)?.message || err}`)); } @@ -2069,6 +2135,8 @@ export async function runAndroid(options: RunAndroidOptions = {} as RunAndroidOp storeKey, waitedForBuild, uploadPending, + providerUpload, + providerName, remote, abandonedRemote, started, diff --git a/packages/stim-cli/src/commands/guide.ts b/packages/stim-cli/src/commands/guide.ts index 3484dbde..a2ff718c 100644 --- a/packages/stim-cli/src/commands/guide.ts +++ b/packages/stim-cli/src/commands/guide.ts @@ -948,7 +948,9 @@ lines Stim composes rather than on files the project owns: Expo's config override on SDK 54+. Expo SDK 53 and older use their normal Metro cache. Turn it off machine-wide with { "caches": { "injectMetroStore": false } } in - ~/.stim/config.json; see \`guide settings\`. + ~/.stim/config.json; see \`guide settings\`. A project that calls + \`sharedCacheStores()\` from @stim-cli/metro in its own metro + config also gets the \`cache.provider\` tier behind that store. Each says so in one dim line. There is nothing to install, wire or commit, and no setup skill to run. \`stim doctor\` is the read-only second opinion when @@ -958,22 +960,33 @@ provider on a key this SDK ignores) plus the project-side settings that matter solely for builds you make OUTSIDE Stim. A clean doctor means there is nothing Stim needs from this repo. -THE BUILD CACHE HAS TWO LEVELS +THE BUILD CACHE HAS THREE LEVELS 1. Stim's own, on this machine: a directory under ~/.stim shared by every worktree, keyed on the @expo/fingerprint hash of the native inputs. - Free, instant, offline, and the only level a bare React Native project - has. - 2. On an EXPO project only, the provider the project ALREADY configured for + Free, instant, offline, and the only level a project without any + provider has. + 2. The project's own cache provider, on ANY project including bare React + Native: \`cache.provider\` in the settings, a module implementing the + @stim-cli/cache contract (see \`guide settings\`). Consulted only when + level one misses, and its hit is stored into level one before install. + The same contract serves the Metro transform cache. + 3. On an EXPO project only, the provider the project ALREADY configured for Expo (\`expo.buildCacheProvider\` -- "eas", or a module of its own). - Consulted only when level one misses, bounded so a slow or expired remote - cannot stall the loop, and a hit is copied into level one on the way past - so the next workspace on this machine gets it for free. After a build, - the result is stored locally AND handed to the provider. + Consulted only when levels one and two miss, bounded so a slow or expired + remote cannot stall the loop, and a hit is copied into level one on the + way past so the next workspace on this machine gets it for free. After a + build, the result is stored locally AND handed to both providers, which + run independently. Stim never configures a provider and never suggests changing one: a project without one is a perfectly ordinary local-only project (doctor does not ask for one either -- a provider only serves builds run OUTSIDE Stim). + A provider that fails to load, times out, or errors produces ONE note per + failure class and the run continues on the local cache. \`gc\` reports, + trims, and clears local caches only: the provider contract has no delete + operation, so no local command can remove data a team or CI system shares. + A MISS explains itself when it can. When this workspace's previous build stored its fingerprint sources beside the cache entry, the fingerprint line gains " -- N sources changed: ", and the full list @@ -1020,12 +1033,12 @@ ONE COMPILE PER FINGERPRINT, ACROSS EVERY WORKSPACE builder that is alive but wedged is the only case a wait can outlive, and that ends after ~90 minutes with STIM_BUILD_WAIT_TIMEOUT naming the lock. - --no-build-cache looks nothing up -- not level one, not level two -- and - takes no lock and never waits, because it asked for a compile of its own. - It still STORES the result, over the entry it was told not to trust, and - still uploads it. Use it when a cached artifact is suspect; the --json - payload reports cacheSkipped: true so a caller can tell that run apart from - a plain miss. + --no-build-cache looks nothing up -- not the local cache, not either + provider -- and takes no lock and never waits, because it asked for a compile + of its own. It still STORES the result, over the entry it was told not to + trust, and still uploads it. Use it when a cached artifact is suspect; the + --json payload reports cacheSkipped: true so a caller can tell that run apart + from a plain miss. OPT-IN CONCURRENCY LIMITS (UNLIMITED BY DEFAULT) Stim imposes NO limits of its own: unset is exactly the behaviour above -- @@ -1457,6 +1470,27 @@ ${ANDROID_AVD_CONFIG_HELP.map((line) => ` ${line}`).joi worktree.exclude additional --carry-ignored skip list, same role as .worktreeexclude. Registered nested Git worktrees are always skipped. + cache.provider one optional SECOND-TIER cache provider: a module + path relative to the settings file that names it, or a + package name. It implements the @stim-cli/cache + contract and can serve Metro transforms, native build + artifacts, or both. The local filesystem stays tier + one; a provider is read only after a local miss and + written after the local write. Failures and timeouts + are cache misses, never build or bundle failures. + Stim ships no provider and never configures one. + This module is EXECUTABLE CODE that every worktree on + this repository runs; review a committed value the way + you review a build script. + \`stim ios\` and \`stim android\` always use it. Metro + uses it only when the project's own metro.config.js + calls \`sharedCacheStores()\` from @stim-cli/metro: the + store Stim injects for you (bare in-process, or the + Expo config override) stays local-only. + cache.options free-form object handed to that module's factory. It + merges key by key across settings layers. Keep secrets + out of the committed file: read them from the + environment or the machine layers. caches extra shared-cache paths for \`gc\` to report. A JSON array; every path is treated as a flat store. diff --git a/packages/stim-cli/src/commands/ios.ts b/packages/stim-cli/src/commands/ios.ts index 227606bd..4a7309bc 100644 --- a/packages/stim-cli/src/commands/ios.ts +++ b/packages/stim-cli/src/commands/ios.ts @@ -4,12 +4,24 @@ import { mkdirSync, openSync, readFileSync, rmSync } from 'node:fs'; import { basename, join } from 'node:path'; import { spawnEntry } from '../spawn-entry.ts'; import { InvalidArgumentError, type Command } from 'commander'; +import { + createWarnOnce, + loadCacheProvider, + resolveTieredBuild, + storeTieredBuild, + type LoadCacheProviderResult, + type ProviderCallResult, +} from '@stim-cli/cache'; import { buildCacheKey, describeFingerprintMiss, + filesystemBuildCapability, fingerprintDiffRecord, fingerprintDiffSuffix, fingerprintProject, + prepareProviderDownloadDir, + providerDownloadPath, + providerUploadOutcome, refingerprintAfterMutation, resolveBuild, storeBuild, @@ -81,13 +93,15 @@ import { getExecutor } from '../exec.ts'; import type { CacheHitLevel, CompilationCacheActivity, IosFacts, RemoteDeviceBackend } from '../types.ts'; import { NOT_OURS_FOREIGN_CWD, isPidAlive, resolveProjectMetro } from '../metro.ts'; import { createNdjsonWriter, type NdjsonWriter } from '../ndjson.ts'; -import { ensureWorkspaceStorage, workspaceLogsDir } from '../paths.ts'; +import { ensureWorkspaceStorage, workspaceDir, workspaceLogsDir } from '../paths.ts'; import { detectBundleId, detectIsExpo, findProjectRoot, isPackageResolvable, projectShortcut } from '../project.ts'; import { + cacheProviderSettingError, publicUrlSetting, REMOTE_DEVICE_BACKENDS, remoteDeviceSettingError, remoteIosSetting, + resolveCacheProviderConfig, resolveSettings, tunnelModeSetting, unknownSettingKeys, @@ -651,6 +665,8 @@ interface IosDeps { untrackedNativeFiles: typeof untrackedNativeFiles; resolveBuild: typeof resolveBuild; storeBuild: typeof storeBuild; + resolveCacheProviderConfig: typeof resolveCacheProviderConfig; + loadCacheProvider: typeof loadCacheProvider; acquireBuildLock: typeof acquireBuildLock; releaseBuildLock: typeof releaseBuildLock; waitForBuild: typeof waitForBuild; @@ -707,6 +723,8 @@ const DEFAULT_DEPS: IosDeps = { untrackedNativeFiles, resolveBuild, storeBuild, + resolveCacheProviderConfig, + loadCacheProvider, acquireBuildLock, releaseBuildLock, waitForBuild, @@ -958,6 +976,7 @@ interface ReportIosResultArgs { waitedForBuild: WaitedForBuild | null; launchState: boolean | string; remote: LoadProjectProviderResult | null; + providerName: string | null; closeWriter: () => void; webPreviewUrl: string | null; } @@ -985,6 +1004,7 @@ function reportIosResult({ waitedForBuild, launchState, remote, + providerName, closeWriter, webPreviewUrl, }: ReportIosResultArgs): IosFacts { @@ -1030,7 +1050,7 @@ function reportIosResult({ const summary = `OK: ${bundleId} on ${deviceLabel(device, udid)}, ` + (release ? `${configuration} (embedded JS, no Metro)` : `Metro port ${metroPort}`) + - ` (${cacheDescription(cacheHit, remote?.name)}, ${formatDuration(durationMs)})`; + ` (${cacheDescription(cacheHit, remote?.name ?? providerName)}, ${formatDuration(durationMs)})`; const outcome = launchState === LAUNCH_UNVERIFIED ? chalk.yellow(`${summary} -- launch UNVERIFIED`) @@ -1038,7 +1058,7 @@ function reportIosResult({ ? chalk.green(`${summary} -- bundle requested, still building`) : chalk.green(summary); const deviceName = device?.deviceName ?? device?.name ?? udid; - const cacheResult = useBuildCache ? cacheDescription(cacheHit, remote?.name) : 'bypassed; built'; + const cacheResult = useBuildCache ? cacheDescription(cacheHit, remote?.name ?? providerName) : 'bypassed; built'; const metroResult = release ? `embedded (${configuration})` : !metroCheck @@ -1087,6 +1107,8 @@ interface FinishIosRunArgs { note: (line: string) => void; logWriter: () => NdjsonWriter; uploadPending: Promise | null; + providerUpload: Promise> | null; + providerName: string | null; remote: LoadProjectProviderResult | null; abandonedRemote: boolean; elapsed: () => number; @@ -1125,6 +1147,8 @@ async function finishIosRun({ note, logWriter, uploadPending, + providerUpload, + providerName, remote, abandonedRemote: remoteWasAbandoned, elapsed, @@ -1238,6 +1262,11 @@ async function finishIosRun({ logWriter().write(launchOutcomeRecord({ launchState, release, bundleId, configuration, metroPort })); const uploadWasAbandoned = await finishIosUpload(uploadPending, remote, phase, note); + const providerOutcome = providerUploadOutcome(providerUpload ? await providerUpload : null, providerName); + if (providerOutcome) { + if (providerOutcome.warn) note(chalk.yellow(phaseLine('cache', providerOutcome.line))); + else phase('cache', providerOutcome.line); + } const facts = reportIosResult({ d, root, @@ -1261,6 +1290,7 @@ async function finishIosRun({ waitedForBuild, launchState, remote, + providerName, closeWriter, webPreviewUrl: remoteDevice?.webPreviewUrl() ?? null, }); @@ -1352,14 +1382,18 @@ export async function runIos(opts: IosCommandOptions = {}, overrides: Partial | null = null; + let providerUpload: Promise> | null = null; + let providerName: string | null = null; + let providerLoad: Promise | null = null; + const cacheWarn = createWarnOnce((line) => note(chalk.yellow(phaseLine('cache', line)))); + const loadProvider = cacheProviderConfig + ? () => (providerLoad ??= d.loadCacheProvider({ projectRoot: root, config: cacheProviderConfig })) + : null; let waitedForBuild: WaitedForBuild | null = null; let swapDir: string | null = null; let buildFailure: BuildFailureFields = {}; @@ -1569,7 +1610,16 @@ export async function runIos(opts: IosCommandOptions = {}, overrides: Partial stored locally' : ''}`); + } + appPath = found?.path ?? null; return true; } @@ -1880,7 +1935,16 @@ export async function runIos(opts: IosCommandOptions = {}, overrides: Partial { diff --git a/packages/stim-cli/src/settings.ts b/packages/stim-cli/src/settings.ts index 94c71d0a..ecfa599a 100644 --- a/packages/stim-cli/src/settings.ts +++ b/packages/stim-cli/src/settings.ts @@ -1,5 +1,6 @@ import { existsSync, readFileSync, realpathSync, statSync } from 'fs'; import { isAbsolute, join, relative, resolve, sep } from 'path'; +import type { CacheProviderConfig } from '@stim-cli/cache'; import { getProjectSettings, getRepoSettings } from './config.ts'; import { TUNNEL_MODES, type TunnelMode } from './engine/metro-reach.ts'; import type { RemoteDeviceBackend, Settings, SettingsObject } from './types.ts'; @@ -45,10 +46,12 @@ const KNOWN_SETTINGS = new Set([ 'worktree.baseRef', 'worktree.include', 'worktree.exclude', + 'cache.provider', + 'cache.options', 'caches', ]); -const OPEN_SETTINGS_OBJECTS = new Set(['android.avdConfig']); +const OPEN_SETTINGS_OBJECTS = new Set(['android.avdConfig', 'cache.options']); export const MIN_ANDROID_DATA_PARTITION_SIZE_GB: number = 6; export const DEFAULT_ANDROID_DATA_PARTITION_SIZE_GB: number = 8; @@ -373,6 +376,67 @@ export function resolveSettings({ ]); } +interface CacheSettingsLayer { + settings: SettingsObject; + baseDir: string | null; +} + +function cacheBlock(layer: SettingsObject | null | undefined): SettingsObject | null { + if (!isPlainObject(layer) || !isPlainObject(layer.cache)) return null; + return layer.cache; +} + +export function resolveCacheProviderConfig({ + projectPath, + gitCommonDir, + repoRoot, +}: { + projectPath?: string | null; + gitCommonDir?: string | null; + repoRoot?: string | null; +}): CacheProviderConfig | null { + const layers: CacheSettingsLayer[] = [ + { settings: projectPath ? getProjectSettings(projectPath) : {}, baseDir: projectPath ?? null }, + { settings: gitCommonDir ? getRepoSettings(gitCommonDir) : {}, baseDir: repoRoot ?? projectPath ?? null }, + { settings: readCommittedSettings(repoRoot), baseDir: repoRoot ?? null }, + ]; + + let provider: string | null = null; + let baseDir: string | null = null; + for (const layer of layers) { + const reference = cacheBlock(layer.settings)?.provider; + if (typeof reference !== 'string' || reference.trim() === '' || layer.baseDir === null) continue; + provider = reference.trim(); + baseDir = layer.baseDir; + break; + } + if (provider === null || baseDir === null) return null; + + const options = mergeSettingsLayers( + layers.map((layer) => { + const block = cacheBlock(layer.settings)?.options; + return isPlainObject(block) ? block : null; + }), + ); + return { provider, options, baseDir }; +} + +export function cacheProviderSettingError(settings: SettingsObject): string | null { + const block = settings.cache; + if (!isPlainObject(block)) { + return block === undefined + ? null + : `Invalid cache setting ${JSON.stringify(block)}. Expected an object with provider and options.`; + } + if ('provider' in block && (typeof block.provider !== 'string' || block.provider.trim() === '')) { + return `Invalid cache.provider setting ${JSON.stringify(block.provider)}. Expected a module path or package name.`; + } + if ('options' in block && !isPlainObject(block.options)) { + return `Invalid cache.options setting ${JSON.stringify(block.options)}. Expected an object.`; + } + return null; +} + export const REMOTE_DEVICE_BACKENDS: readonly RemoteDeviceBackend[] = ['proxy', 'eas'] as const; export function remoteIosSetting(settings: SettingsObject): RemoteDeviceBackend | null { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f10905f..15a1ad9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -131,6 +131,8 @@ importers: specifier: 4.1.11 version: 4.1.11(@types/node@26.3.0)(vite@8.2.2(@types/node@26.3.0)(jiti@2.7.0)(terser@5.50.0)(yaml@2.9.0)) + packages/cache: {} + packages/core: {} packages/expo-build-cache: @@ -141,6 +143,9 @@ importers: packages/metro: dependencies: + '@stim-cli/cache': + specifier: ^1.0.0-rc.4 + version: link:../cache '@stim-cli/core': specifier: ^1.0.0-rc.4 version: link:../core @@ -153,6 +158,9 @@ importers: '@expo/fingerprint': specifier: ^0.20.10 version: 0.20.10 + '@stim-cli/cache': + specifier: ^1.0.0-rc.4 + version: link:../cache '@stim-cli/core': specifier: ^1.0.0-rc.4 version: link:../core diff --git a/test/runtime-floor.mjs b/test/runtime-floor.mjs index cff68e10..a55ac550 100644 --- a/test/runtime-floor.mjs +++ b/test/runtime-floor.mjs @@ -7,7 +7,7 @@ import { pathToFileURL } from 'node:url'; const repositoryRoot = join(import.meta.dirname, '..'); const require = createRequire(join(repositoryRoot, 'packages', 'stim-cli', 'package.json')); -const packageDirs = ['stim-cli', 'core', 'metro', 'expo-build-cache']; +const packageDirs = ['stim-cli', 'core', 'cache', 'metro', 'expo-build-cache']; for (const directory of packageDirs) { const root = join(repositoryRoot, 'packages', directory); @@ -25,6 +25,7 @@ for (const directory of packageDirs) { const entrypoints = [ ['@stim-cli/core', 'configDir'], + ['@stim-cli/cache', 'loadCacheProvider'], ['@stim-cli/expo-build-cache', 'cacheRoot'], ['@stim-cli/metro', 'sharedCacheStores'], ['stim-cli/cache-manifest', 'readManifest'], diff --git a/website/docs/settings.md b/website/docs/settings.md index 602815da..fa612c64 100644 --- a/website/docs/settings.md +++ b/website/docs/settings.md @@ -45,11 +45,20 @@ keys produce a warning. | `worktree.baseRef` | Default worktree base: `head`, `fresh`, or a git ref | | `worktree.include` | Explicit ignored paths to carry | | `worktree.exclude` | Ignored paths skipped by `--carry-ignored` | +| `cache.provider` | Optional second-tier cache provider module | +| `cache.options` | Options passed to that provider | | `caches` | Additional cache paths reported by `gc` | Do not put secrets in a committed `.stim.json`. Keep secrets in ignored files and carry those files into a worktree. +`cache.provider` names a module that Stim executes in every worktree on the +repository. Review a committed value the way you review a build script, and +keep provider credentials in the environment or in machine settings. Stim reads +the module for `stim ios` and `stim android`; Metro uses it only when the +project's own `metro.config.js` calls `sharedCacheStores()` from +`@stim-cli/metro`. + ### Android AVD overrides `android.avdConfigFile` reads an Android `config.ini` file. `android.avdConfig`