diff --git a/.github/shared/package.json b/.github/shared/package.json index c4ec4fa08407..fe9837727e5a 100644 --- a/.github/shared/package.json +++ b/.github/shared/package.json @@ -14,6 +14,7 @@ "./exec": "./src/exec.ts", "./git": "./src/git.ts", "./github": "./src/github.ts", + "./glob": "./src/glob.ts", "./logger": "./src/logger.ts", "./markdown": "./src/markdown.ts", "./math": "./src/math.ts", @@ -34,6 +35,7 @@ "./testing": "./src/testing.ts", "./time": "./src/time.ts", "./typespec-metadata": "./src/typespec-metadata.ts", + "./typespec-projects": "./src/typespec-projects.ts", "./test/examples": "./test/examples.ts" }, "bin": { diff --git a/.github/shared/readme.md b/.github/shared/readme.md index 48f481136eb8..0786d911ca5b 100644 --- a/.github/shared/readme.md +++ b/.github/shared/readme.md @@ -153,6 +153,11 @@ function main() { - `defaultLogger` — shared non-debug logger instance. - `debugLogger` — shared debug-enabled logger instance. +### `glob` — filesystem discovery + +- `globFiles(pattern, options)` — native glob discovery restricted to files, with optional `cwd` + and exclusion patterns. Includes file symlinks without traversing directory symlinks. + ### `math` — math helpers - `toPercent(value, decimals)` — format a `0..1` ratio as a percentage string. @@ -285,6 +290,21 @@ so the assertion still catches whitespace changes in the actual output. - `formatDuration(ms)` — human-readable duration string. - `getDuration(from, to)` — milliseconds between two dates. +### `typespec-projects` — TypeSpec project discovery + +- `findTypeSpecProjects(root)` — sorted, unique absolute project directories beneath `root`. + Discovers `tspconfig.*`, including invalid extensions so validation can report them, and excludes + `node_modules`. +- `findTypeSpecProjectsInChangedServices(repoRoot, changedFiles, logger?)` — discovers every project + in each `specification//` touched by repository-relative POSIX changed paths. Includes + sibling and nested projects for any file type, including deleted paths. Missing services are + skipped with a debug message; other filesystem failures propagate. + +These helpers do not fetch changed files, inspect imports, pair Swagger READMEs, or apply a +core-file/all-project fallback. Callers own those policies. Pass both old and new paths when a +rename must select both services. Results are directories, not `tspconfig.yaml` paths; consumers +requiring valid configs must account for invalid extensions. + ### `typespec-metadata` — TypeSpec SDK metadata - `generateTypeSpecMetadata(folder, options)` — run the `@azure-tools/typespec-metadata` emitter, diff --git a/eng/tools/typespec-validation/src/glob.ts b/.github/shared/src/glob.ts similarity index 100% rename from eng/tools/typespec-validation/src/glob.ts rename to .github/shared/src/glob.ts diff --git a/.github/shared/src/typespec-projects.ts b/.github/shared/src/typespec-projects.ts new file mode 100644 index 000000000000..74ccb9ac7d8a --- /dev/null +++ b/.github/shared/src/typespec-projects.ts @@ -0,0 +1,57 @@ +import { stat } from "node:fs/promises"; +import { dirname, relative, resolve } from "pathe"; +import { globFiles } from "./glob.ts"; +import { defaultLogger, type ILogger } from "./logger.ts"; + +/** Finds sorted, unique absolute project directories, including invalid config extensions. */ +export async function findTypeSpecProjects(root: string): Promise { + const configs = await globFiles("**/tspconfig.*", { + cwd: root, + exclude: ["**/node_modules/**"], + }); + const folders = [...new Set(configs.map((config) => dirname(config)))].sort(); + return folders.map((folder) => resolve(root, folder)); +} + +/** + * Finds every project in services touched by repository-relative POSIX changed paths. + * Includes sibling and nested projects even for deleted files; missing services are skipped. + * Git comparison and core-file/all-project policies belong to the caller. + */ +export async function findTypeSpecProjectsInChangedServices( + root: string, + changedFiles: readonly string[], + logger: ILogger = defaultLogger, +): Promise { + const directories = [ + ...new Set( + changedFiles.flatMap((file) => { + const match = /^specification\/[^/]+\//.exec(file); + return match ? [match[0].slice(0, -1)] : []; + }), + ), + ].sort(); + const projects = new Set(); + for (const directory of directories) { + const folder = resolve(root, directory); + let isDirectory: boolean; + try { + isDirectory = (await stat(folder)).isDirectory(); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + isDirectory = false; + } else { + throw error; + } + } + if (!isDirectory) { + logger.debug(`Cannot find directory ${directory}`); + continue; + } + for (const project of await findTypeSpecProjects(folder)) { + projects.add(relative(root, project)); + } + } + + return [...projects].sort().map((project) => resolve(root, project)); +} diff --git a/eng/tools/typespec-validation/test/glob.test.ts b/.github/shared/test/glob.test.ts similarity index 100% rename from eng/tools/typespec-validation/test/glob.test.ts rename to .github/shared/test/glob.test.ts diff --git a/.github/shared/test/typespec-projects.test.ts b/.github/shared/test/typespec-projects.test.ts new file mode 100644 index 000000000000..6ea9c8307f14 --- /dev/null +++ b/.github/shared/test/typespec-projects.test.ts @@ -0,0 +1,114 @@ +import * as fs from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "pathe"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ConsoleLogger } from "../src/logger.ts"; +import { + findTypeSpecProjects, + findTypeSpecProjectsInChangedServices, +} from "../src/typespec-projects.ts"; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, stat: vi.fn(actual.stat) }; +}); + +let root: string; + +async function addProject(path: string, config = "tspconfig.yaml") { + const folder = join(root, path); + await fs.mkdir(folder, { recursive: true }); + await fs.writeFile(join(folder, config), ""); + return folder; +} + +beforeEach(async () => { + root = await fs.mkdtemp(join(tmpdir(), "specs-shared-typespec-projects-")); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +describe("findTypeSpecProjects", () => { + it("discovers sorted, unique projects including invalid config extensions, not directories or dependencies", async () => { + const last = await addProject("z"); + const first = await addProject("a", "tspconfig.yml"); + await addProject("a"); + const nested = await addProject("a/nested"); + const afterNested = await addProject("aZ"); + await addProject("node_modules/dependency"); + await addProject("a/node_modules/dependency"); + await fs.mkdir(join(root, "not-a-project", "tspconfig.yaml"), { recursive: true }); + + await expect(findTypeSpecProjects(root)).resolves.toEqual([first, nested, afterNested, last]); + }); +}); + +describe("findTypeSpecProjectsInChangedServices", () => { + it("selects every project in affected services for nested models, shared libraries, and Swagger changes", async () => { + const parent = await addProject("specification/first/Project"); + const nested = await addProject("specification/first/Project/models/Nested"); + const sibling = await addProject("specification/first/Sibling"); + const management = await addProject("specification/second/Service.Management"); + const dataPlane = await addProject("specification/second/Service"); + const swaggerProject = await addProject("specification/third/Project"); + await addProject("specification/unaffected/Other"); + + await expect( + findTypeSpecProjectsInChangedServices(root, [ + "specification/third/resource-manager/stable/v1/service.json", + "specification/first/Project/models/model.tsp", + "specification/first/Project/client.tsp", + "specification/second/Service.Shared/model.tsp", + "documentation/readme.md", + "specification/README.md", + "specification-other/service/main.tsp", + "package.json", + ]), + ).resolves.toEqual([parent, nested, sibling, dataPlane, management, swaggerProject]); + }); + + it("includes surviving services for deleted paths and reports missing or non-directory services", async () => { + const existing = await addProject("specification/service/Existing"); + await fs.writeFile(join(root, "specification/not-directory"), ""); + vi.spyOn(console, "debug").mockImplementation(() => {}); + + await expect( + findTypeSpecProjectsInChangedServices( + root, + [ + "specification/service/Deleted/models/deleted.tsp", + "specification/removed/Project/main.tsp", + "specification/removed/Project/tspconfig.yaml", + "specification/not-directory/main.tsp", + ], + new ConsoleLogger(true), + ), + ).resolves.toEqual([existing]); + expect(vi.mocked(console.debug).mock.calls).toEqual([ + ["Cannot find directory specification/not-directory"], + ["Cannot find directory specification/removed"], + ]); + }); + + it("returns an empty selection when no files changed", async () => { + await expect(findTypeSpecProjectsInChangedServices(root, [])).resolves.toEqual([]); + }); + + it.each([ + Object.assign(new Error("Permission denied"), { code: "EACCES" }), + new Error("Unexpected filesystem failure"), + "Unexpected rejection", + ])( + "propagates filesystem failures instead of returning an empty selection: %s", + async (error) => { + vi.mocked(fs.stat).mockRejectedValueOnce(error); + + await expect( + findTypeSpecProjectsInChangedServices(root, ["specification/service/Project/main.tsp"]), + ).rejects.toBe(error); + }, + ); +}); diff --git a/eng/tools/typespec-validation/src/find-projects.ts b/eng/tools/typespec-validation/src/find-projects.ts index e5292289a03f..99867a99232f 100644 --- a/eng/tools/typespec-validation/src/find-projects.ts +++ b/eng/tools/typespec-validation/src/find-projects.ts @@ -1,8 +1,10 @@ import { getChangedFiles } from "@azure-tools/specs-shared/changed-files"; import type { ILogger } from "@azure-tools/specs-shared/logger"; -import { stat } from "node:fs/promises"; -import { dirname, join, relative, resolve } from "pathe"; -import { globFiles } from "./glob.ts"; +import { + findTypeSpecProjects, + findTypeSpecProjectsInChangedServices, +} from "@azure-tools/specs-shared/typespec-projects"; +import { join } from "pathe"; export interface ChangedProjectsOptions { baseCommitish: string; @@ -33,16 +35,6 @@ function isCoreFile(file: string): boolean { ); } -/** Find project folders, including invalid config extensions so validation can report them. */ -export async function findProjects(root: string): Promise { - const configs = await globFiles("**/tspconfig.*", { - cwd: root, - exclude: ["**/node_modules/**"], - }); - const folders = [...new Set(configs.map((config) => dirname(config)))].sort(); - return folders.map((folder) => resolve(root, folder)); -} - export async function findChangedProjects( root: string, options: ChangedProjectsOptions, @@ -59,45 +51,13 @@ export async function findChangedProjects( if (!options.ignoreCoreFiles && changedFiles.some(isCoreFile)) { console.log("Found changes to core eng or root files so checking all specs."); return { - projects: await findProjects(join(root, "specification")), + projects: await findTypeSpecProjects(join(root, "specification")), checkingAllSpecs: true, }; } - // Match Get-TypeSpec-Folders.ps1: select the entire service folder, including - // sibling and nested projects. Deleted files still affect surviving services. - const directories = [ - ...new Set( - changedFiles.flatMap((file) => { - const match = /^specification\/[^/]+\//.exec(file); - return match ? [match[0].slice(0, -1)] : []; - }), - ), - ].sort(); - const projects = new Set(); - for (const directory of directories) { - const folder = resolve(root, directory); - let isDirectory: boolean; - try { - isDirectory = (await stat(folder)).isDirectory(); - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOENT") { - isDirectory = false; - } else { - throw error; - } - } - if (!isDirectory) { - options.logger.debug(`Cannot find directory ${directory}`); - continue; - } - for (const project of await findProjects(folder)) { - projects.add(relative(root, project)); - } - } - return { - projects: [...projects].sort().map((project) => resolve(root, project)), + projects: await findTypeSpecProjectsInChangedServices(root, changedFiles, options.logger), checkingAllSpecs: false, }; } diff --git a/eng/tools/typespec-validation/src/rules/compile.ts b/eng/tools/typespec-validation/src/rules/compile.ts index d172a9d09a2d..52a03f34e5b3 100644 --- a/eng/tools/typespec-validation/src/rules/compile.ts +++ b/eng/tools/typespec-validation/src/rules/compile.ts @@ -1,4 +1,5 @@ import { filterAsync } from "@azure-tools/specs-shared/array"; +import { globFiles } from "@azure-tools/specs-shared/glob"; import { untilLastSegmentWithParent } from "@azure-tools/specs-shared/path"; import { getRootFolder } from "@azure-tools/specs-shared/simple-git"; import type { ILogger } from "@azure-tools/specs-shared/logger"; @@ -7,7 +8,6 @@ import { stripVTControlCharacters } from "node:util"; import path, { basename, dirname, normalize } from "pathe"; import { reportCommandOutput } from "../command-output.ts"; import { blocks, filePath, indent, lines, verbatim } from "../diagnostic-content.ts"; -import { globFiles } from "../glob.ts"; import { type Diagnostic, type RuleResult } from "../rule-result.ts"; import { type Rule } from "../rule.ts"; import { diff --git a/eng/tools/typespec-validation/src/rules/folder-structure.ts b/eng/tools/typespec-validation/src/rules/folder-structure.ts index 1ad5c5752792..182804dc5f1d 100644 --- a/eng/tools/typespec-validation/src/rules/folder-structure.ts +++ b/eng/tools/typespec-validation/src/rules/folder-structure.ts @@ -1,8 +1,8 @@ +import { globFiles } from "@azure-tools/specs-shared/glob"; import type { ILogger } from "@azure-tools/specs-shared/logger"; import { readFile } from "node:fs/promises"; import path from "pathe"; import { simpleGit } from "simple-git"; -import { globFiles } from "../glob.ts"; import { failure, type Diagnostic, type RuleResult } from "../rule-result.ts"; import { type Rule } from "../rule.ts"; import { parse } from "../tsp-config.ts"; diff --git a/eng/tools/typespec-validation/src/run-projects.ts b/eng/tools/typespec-validation/src/run-projects.ts index a6ae98b8f578..3692e4d09452 100644 --- a/eng/tools/typespec-validation/src/run-projects.ts +++ b/eng/tools/typespec-validation/src/run-projects.ts @@ -1,5 +1,6 @@ import { ConsoleLogger } from "@azure-tools/specs-shared/logger"; import { getRootFolder } from "@azure-tools/specs-shared/simple-git"; +import { findTypeSpecProjects } from "@azure-tools/specs-shared/typespec-projects"; import { getSuppressions } from "@azure-tools/suppressions"; import { spawn } from "node:child_process"; import { appendFile, stat } from "node:fs/promises"; @@ -8,7 +9,7 @@ import { fileURLToPath } from "node:url"; import pc from "picocolors"; import { simpleGit } from "simple-git"; import { formatRuleSummary, supportsColor, type RuleCounts } from "./diagnostics.ts"; -import { findChangedProjects, findProjects, type ChangedProjectsOptions } from "./find-projects.ts"; +import { findChangedProjects, type ChangedProjectsOptions } from "./find-projects.ts"; import { cleanWorktree } from "./git-cleanup.ts"; interface RunOptions { @@ -55,7 +56,7 @@ export async function runAll( throw new Error(`Please run TypeSpec Validation on a directory path: ${root}`); } - let projects = await findProjects(root); + let projects = await findTypeSpecProjects(root); if (projects.length === 0) { console.error(`No TypeSpec projects found in ${root}`); return false; diff --git a/eng/tools/typespec-validation/test/compile.test.ts b/eng/tools/typespec-validation/test/compile.test.ts index d890536a6e98..f1d1dc770923 100644 --- a/eng/tools/typespec-validation/test/compile.test.ts +++ b/eng/tools/typespec-validation/test/compile.test.ts @@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } fr import * as fsPromises from "node:fs/promises"; import path from "pathe"; -import * as nativeGlob from "../src/glob.ts"; +import * as nativeGlob from "@azure-tools/specs-shared/glob"; import { CompileRule } from "../src/rules/compile.ts"; import { diagnosticDetails } from "./diagnostics.ts"; diff --git a/eng/tools/typespec-validation/test/folder-structure.test.ts b/eng/tools/typespec-validation/test/folder-structure.test.ts index 64534e5666da..2b1c0506082a 100644 --- a/eng/tools/typespec-validation/test/folder-structure.test.ts +++ b/eng/tools/typespec-validation/test/folder-structure.test.ts @@ -6,7 +6,7 @@ import { contosoTspConfig } from "@azure-tools/specs-shared/test/examples"; import { strict as assert } from "node:assert"; import { simpleGit } from "simple-git"; import { afterEach, beforeEach, describe, it, type MockInstance, vi } from "vitest"; -import * as nativeGlob from "../src/glob.ts"; +import * as nativeGlob from "@azure-tools/specs-shared/glob"; import { FolderStructureRule } from "../src/rules/folder-structure.ts"; import * as utils from "../src/utils.ts"; diff --git a/eng/tools/typespec-validation/test/mocks.ts b/eng/tools/typespec-validation/test/mocks.ts index 13c19a2c7903..c118f4f5d656 100644 --- a/eng/tools/typespec-validation/test/mocks.ts +++ b/eng/tools/typespec-validation/test/mocks.ts @@ -6,7 +6,7 @@ vi.mock("node:fs/promises", () => ({ readFile: vi.fn().mockResolvedValue('{"info": {"x-typespec-generated": true}}'), })); -vi.mock("../src/glob.ts", () => ({ +vi.mock("@azure-tools/specs-shared/glob", () => ({ globFiles: vi.fn().mockResolvedValue([]), }));