feat(meta): add build.nativeDependencies for packages that cannot be bundled

src/meta/__tests__/build.test.ts

@@ -7,6 +7,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest
 
 import { buildProd } from '../build.js';
 import * as externalPluginModule from '../plugins/external.js';
+import * as nativeDependenciesPluginModule from '../plugins/native-dependencies.js';
 
 import type { UserConfigOptions } from '../../user-config.js';
 
@@ -154,6 +155,84 @@ describe('buildProd', () => {
     vi.doUnmock('esbuild');
   });
 
+  it('should not register the native dependencies plugin when none are declared', { timeout: 10000 }, async () => {
+    const nativeDependenciesPlugin = vi.spyOn(nativeDependenciesPluginModule, 'nativeDependenciesPlugin');
+    const outfile = path.join(outdir, 'module-no-native.js');
+    loadUserConfig.mockReturnValue(
+      okAsync({
+        build: {
+          mode: 'module',
+          nativeDependencies: [],
+          outfile
+        },
+        entry: vi.fn()
+      } satisfies UserConfigOptions)
+    );
+    parseEntryFromFunction.mockReturnValueOnce(ok('./example/app.js'));
+    const result = await buildProd({ configFile });
+    expect(result.isOk()).toBe(true);
+    expect(nativeDependenciesPlugin).not.toHaveBeenCalled();
+  });
+
+  it('should emit a declared native dependency beside the bundle', { timeout: 10000 }, async () => {
+    const artifact = path.join(outdir, 'artifact-source');
+    await fs.promises.writeFile(artifact, 'native artifact');
+    const outfile = path.join(outdir, 'module-native.js');
+    loadUserConfig.mockReturnValue(
+      okAsync({
+        build: {
+          mode: 'module',
+          // `neverthrow` stands in for a native package: it is already in the example app's graph, so
+          // the plugin observes a real resolution rather than a stubbed one.
+          nativeDependencies: [
+            {
+              locate: () => artifact,
+              outputName: 'artifact',
+              packageName: 'neverthrow',
+              runtimeEnvVar: 'ARTIFACT_BINARY_PATH'
+            }
+          ],
+          outfile
+        },
+        entry: vi.fn()
+      } satisfies UserConfigOptions)
+    );
+    parseEntryFromFunction.mockReturnValueOnce(ok('./example/app.js'));
+
+    const result = await buildProd({ configFile });
+
+    expect(result.isOk()).toBe(true);
+    expect(fs.existsSync(path.join(outdir, 'artifact'))).toBe(true);
+    expect(fs.readFileSync(outfile, 'utf-8')).toContain('process.env.ARTIFACT_BINARY_PATH ??=');
+  });
+
+  it('should fail the build when a declared native dependency is never imported', async () => {
+    const outfile = path.join(outdir, 'module-missing-native.js');
+    loadUserConfig.mockReturnValue(
+      okAsync({
+        build: {
+          mode: 'module',
+          nativeDependencies: [
+            {
+              locate: () => '/dev/null',
+              outputName: 'absent',
+              packageName: '@scope/never-imported',
+              runtimeEnvVar: 'ABSENT_BINARY_PATH'
+            }
+          ],
+          outfile
+        },
+        entry: vi.fn()
+      } satisfies UserConfigOptions)
+    );
+    parseEntryFromFunction.mockReturnValueOnce(ok('./example/app.js'));
+
+    const result = await buildProd({ configFile });
+
+    expect(result.isErr()).toBe(true);
+    expect(result).toMatchObject({ error: { message: 'Failed to build application' } });
+  });
+
   it('should bundle with bundle:false to mark node_modules as external', { timeout: 10000 }, async () => {
     const externalPlugin = vi.spyOn(externalPluginModule, 'externalPlugin');
     const outfile = path.join(outdir, 'module-unbundled.js');

src/meta/__tests__/native-dependencies.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+  KNOWN_NATIVE_DEPENDENCIES,
+  KNOWN_NATIVE_DEPENDENCY_NAMES,
+  resolveNativeDependency
+} from '../native-dependencies.js';
+
+import type { NativeDependency } from '../../user-config.js';
+
+describe('resolveNativeDependency', () => {
+  it('should expand a known name to its built-in recipe', () => {
+    expect(resolveNativeDependency('esbuild')).toBe(KNOWN_NATIVE_DEPENDENCIES.esbuild);
+  });
+
+  it('should pass a custom dependency through unchanged', () => {
+    const custom: NativeDependency = {
+      locate: () => '/artifact',
+      outputName: 'artifact',
+      packageName: 'custom',
+      runtimeEnvVar: 'CUSTOM_BINARY_PATH'
+    };
+    expect(resolveNativeDependency(custom)).toBe(custom);
+  });
+});
+
+describe('KNOWN_NATIVE_DEPENDENCY_NAMES', () => {
+  it('should list every built-in recipe, so the config validator accepts each one', () => {
+    expect(KNOWN_NATIVE_DEPENDENCY_NAMES).toStrictEqual(Object.keys(KNOWN_NATIVE_DEPENDENCIES));
+  });
+});
+
+describe('esbuild recipe', () => {
+  it('should resolve the executable of the platform package for the current host', () => {
+    const resolve = vi.fn().mockReturnValue('/pkgs/@esbuild/target/bin/esbuild');
+
+    const artifact = KNOWN_NATIVE_DEPENDENCIES.esbuild.locate({
+      entryPath: '/pkgs/esbuild/lib/main.js',
+      require: { resolve } as any
+    });
+
+    expect(resolve).toHaveBeenCalledWith(`@esbuild/${process.platform}-${process.arch}/bin/esbuild`);
+    expect(artifact).toBe('/pkgs/@esbuild/target/bin/esbuild');
+  });
+});

src/meta/build.ts

@@ -4,9 +4,11 @@ import { RuntimeException } from '@douglasneuroinformatics/libjs';
 import { fromAsyncThrowable, ok, ResultAsync } from 'neverthrow';
 
 import { loadUserConfig } from './load.js';
+import { resolveNativeDependency } from './native-dependencies.js';
 import { parseEntryFromFunction } from './parse.js';
 import { docsPlugin } from './plugins/docs.js';
 import { externalPlugin } from './plugins/external.js';
+import { nativeDependenciesPlugin } from './plugins/native-dependencies.js';
 import { prismaPlugin } from './plugins/prisma.js';
 import { swcPlugin } from './plugins/swc.js';
 
@@ -55,6 +57,10 @@ export function buildProd({
         plugins.push(externalPlugin());
       }
 
+      if (config.build.nativeDependencies?.length) {
+        plugins.push(nativeDependenciesPlugin(config.build.nativeDependencies.map(resolveNativeDependency)));
+      }
+
       await esbuild.build({
         banner: {
           js: "Object.defineProperties(globalThis, { __dirname: { value: import.meta.dirname, writable: false }, __filename: { value: import.meta.filename, writable: false }, require: { value: (await import('module')).createRequire(import.meta.url), writable: false } });"

src/meta/load.ts

@@ -7,18 +7,26 @@ import { z } from 'zod/v4';
 
 import { AbstractAppContainer } from '../app/app.base.js';
 import { importDefault } from './import.js';
+import { KNOWN_NATIVE_DEPENDENCY_NAMES } from './native-dependencies.js';
 import { parseEntryFromFunction } from './parse.js';
 
 import type { UserConfigOptions } from '../user-config.js';
 
 // we cannot use zod function here as we cannot have any wrappers apply and screw up toString representation
 const $AnyFunction = z.custom<(...args: any[]) => any>((arg) => typeof arg === 'function', 'must be function');
 
+const $NativeDependency = z.object({
+  locate: $AnyFunction,
+  outputName: z.string().min(1),
+  packageName: z.string().min(1),
+  runtimeEnvVar: z.string().regex(/^[A-Z_][A-Z0-9_]*$/, 'must be a valid environment variable name')
+});
+
 const $UserConfigOptions: z.ZodType<UserConfigOptions> = z.object({
   build: z.object({
     bundle: z.boolean().optional(),
-    esbuildOptions: z.record(z.string(), z.any()).optional(),
     mode: z.enum(['module', 'server']).optional(),
+    nativeDependencies: z.array(z.union([z.enum(KNOWN_NATIVE_DEPENDENCY_NAMES), $NativeDependency])).optional(),
     onComplete: $AnyFunction.optional(),
     outfile: z.string().min(1)
   }),

src/meta/native-dependencies.ts

@@ -0,0 +1,31 @@
+import type { KnownNativeDependencyName, NativeDependency } from '../user-config.js';
+
+/**
+ * Built-in recipes, so naming the package is enough for the common cases.
+ *
+ * Each `locate` resolves through the `require` of the module that imported the package, never through
+ * `libnest`'s own. The application and `libnest` routinely resolve different versions of the same
+ * dependency, and pairing an artifact with a different version of its own JavaScript fails at runtime.
+ */
+const KNOWN_NATIVE_DEPENDENCIES = {
+  esbuild: {
+    // The executable lives in a per-platform sibling package listed in esbuild's optionalDependencies.
+    // `ESBUILD_BINARY_PATH` both points esbuild at it and suppresses esbuild's refusal to run from a
+    // bundle, which it otherwise detects by comparing __filename against its own layout.
+    locate: ({ require }): string => require.resolve(`@esbuild/${process.platform}-${process.arch}/bin/esbuild`),
+    outputName: 'esbuild',
+    packageName: 'esbuild',
+    runtimeEnvVar: 'ESBUILD_BINARY_PATH'
+  }
+} satisfies { [K in KnownNativeDependencyName]: NativeDependency };
+
+const KNOWN_NATIVE_DEPENDENCY_NAMES = Object.keys(KNOWN_NATIVE_DEPENDENCIES) as [
+  KnownNativeDependencyName,
+  ...KnownNativeDependencyName[]
+];
+
+function resolveNativeDependency(dependency: KnownNativeDependencyName | NativeDependency): NativeDependency {
+  return typeof dependency === 'string' ? KNOWN_NATIVE_DEPENDENCIES[dependency] : dependency;
+}
+
+export { KNOWN_NATIVE_DEPENDENCIES, KNOWN_NATIVE_DEPENDENCY_NAMES, resolveNativeDependency };

src/meta/plugins/__tests__/native-dependencies.test.ts

@@ -0,0 +1,158 @@
+import * as path from 'node:path';
+
+import type { PluginBuild } from 'esbuild';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { nativeDependenciesPlugin } from '../native-dependencies.js';
+
+import type { NativeDependency } from '../../../user-config.js';
+
+const fs = vi.hoisted(() => ({
+  chmod: vi.fn(),
+  copyFile: vi.fn()
+}));
+
+vi.mock('node:fs/promises', () => fs);
+
+const createRequire = vi.hoisted(() => vi.fn());
+
+vi.mock('node:module', () => ({ createRequire }));
+
+const ENTRY_PATH = '/pkgs/widget/lib/main.js';
+const ARTIFACT_PATH = '/pkgs/widget-linux-x64/bin/widget';
+
+const dependency: NativeDependency = {
+  locate: vi.fn(() => ARTIFACT_PATH),
+  outputName: 'widget',
+  packageName: 'widget',
+  runtimeEnvVar: 'WIDGET_BINARY_PATH'
+};
+
+/**
+ * A stub of the subset of `PluginBuild` the plugin touches, exposing the two registered callbacks so a
+ * test can drive resolution and completion directly.
+ */
+function createBuild({ resolvedPath = ENTRY_PATH }: { resolvedPath?: string } = {}) {
+  const build = {
+    initialOptions: {
+      banner: { js: '' },
+      outdir: '/app'
+    },
+    onEnd: vi.fn(),
+    onResolve: vi.fn(),
+    resolve: vi.fn().mockResolvedValue({ path: resolvedPath })
+  } satisfies Partial<{ [K in keyof PluginBuild]: any }>;
+  return {
+    build,
+    onEnd: () => build.onEnd.mock.lastCall![0] as () => Promise<void>,
+    onResolve: () =>
+      build.onResolve.mock.lastCall![1] as (args: { kind: string; path: string; resolveDir: string }) => Promise<null>
+  };
+}
+
+const resolveArgs = (specifier: string) => ({ kind: 'import-statement', path: specifier, resolveDir: '/app' });
+
+describe('nativeDependenciesPlugin', () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should assign the environment variable without overwriting one supplied by the operator', async () => {
+    const { build } = createBuild();
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+    expect(build.initialOptions.banner.js).toBe("process.env.WIDGET_BINARY_PATH ??= import.meta.dirname + '/widget';");
+  });
+
+  it('should terminate its banner statement, so a second appending plugin still parses', async () => {
+    const { build } = createBuild();
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+    expect(build.initialOptions.banner.js.endsWith(';')).toBe(true);
+  });
+
+  it('should emit the artifact beside the bundle and make it executable', async () => {
+    const { build, onEnd, onResolve } = createBuild();
+    createRequire.mockReturnValue({ resolve: vi.fn() });
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+
+    await onResolve()(resolveArgs('widget'));
+    await onEnd()();
+
+    expect(dependency.locate).toHaveBeenCalledWith(expect.objectContaining({ entryPath: ENTRY_PATH }));
+    expect(fs.copyFile).toHaveBeenCalledExactlyOnceWith(ARTIFACT_PATH, path.join('/app', 'widget'));
+    expect(fs.chmod).toHaveBeenCalledExactlyOnceWith(path.join('/app', 'widget'), 0o755);
+  });
+
+  it('should write beside the bundle when the output is an outfile rather than an outdir', async () => {
+    const { build, onEnd, onResolve } = createBuild();
+    createRequire.mockReturnValue({ resolve: vi.fn() });
+    const outfileBuild = {
+      ...build,
+      initialOptions: { banner: { js: '' }, outfile: '/dist/server.js' }
+    };
+    await nativeDependenciesPlugin([dependency]).setup(outfileBuild as any);
+
+    await onResolve()(resolveArgs('widget'));
+    await onEnd()();
+
+    expect(fs.copyFile).toHaveBeenCalledExactlyOnceWith(ARTIFACT_PATH, path.join('/dist', 'widget'));
+  });
+
+  it('should locate the artifact through a require rooted at the importing module, not at libnest', async () => {
+    const { build, onEnd, onResolve } = createBuild();
+    const requireFn = { resolve: vi.fn() };
+    createRequire.mockReturnValue(requireFn);
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+
+    await onResolve()(resolveArgs('widget'));
+    await onEnd()();
+
+    expect(createRequire).toHaveBeenCalledWith(ENTRY_PATH);
+    expect(dependency.locate).toHaveBeenCalledWith({ entryPath: ENTRY_PATH, require: requireFn });
+  });
+
+  it('should match a subpath import of a declared dependency', async () => {
+    const { build, onResolve } = createBuild();
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+
+    await onResolve()(resolveArgs('widget/lib/main.js'));
+
+    expect(build.resolve).toHaveBeenCalledOnce();
+  });
+
+  it('should ignore an import that is not a declared dependency', async () => {
+    const { build, onResolve } = createBuild();
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+
+    await expect(onResolve()(resolveArgs('widgetry'))).resolves.toBeNull();
+    expect(build.resolve).not.toHaveBeenCalled();
+  });
+
+  it('should resolve a dependency only once however many times it is imported', async () => {
+    const { build, onResolve } = createBuild();
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+
+    await onResolve()(resolveArgs('widget'));
+    await onResolve()(resolveArgs('widget'));
+
+    expect(build.resolve).toHaveBeenCalledOnce();
+  });
+
+  it('should throw when a declared dependency is never imported, rather than emitting a bundle pointing at nothing', async () => {
+    const { build, onEnd } = createBuild();
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+
+    await expect(onEnd()()).rejects.toThrowError(
+      "Declared native dependency 'widget' was never imported by the application, so its native artifact cannot be located"
+    );
+    expect(fs.copyFile).not.toHaveBeenCalled();
+  });
+
+  it('should throw when a declared dependency cannot be resolved, so an unresolved import is not silently skipped', async () => {
+    const { build, onEnd, onResolve } = createBuild({ resolvedPath: '' });
+    await nativeDependenciesPlugin([dependency]).setup(build as any);
+
+    await onResolve()(resolveArgs('widget'));
+
+    await expect(onEnd()()).rejects.toThrowError("Declared native dependency 'widget' was never imported");
+  });
+});

src/meta/plugins/__tests__/prisma.test.ts

@@ -63,6 +63,12 @@ describe('prismaPlugin', () => {
     );
   });
 
+  it('should terminate its banner statement, so a second appending plugin still parses', async () => {
+    fs.readdir.mockResolvedValueOnce([mockTargetFile]);
+    await prismaPlugin().setup(build as any);
+    expect(build.initialOptions.banner.js.endsWith(';')).toBe(true);
+  });
+
   it('should copy the binary to the target directory', async () => {
     fs.readdir.mockResolvedValueOnce([mockTargetFile]);
     const plugin = prismaPlugin();