diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index 574dd97c..26c48647 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -19,6 +19,7 @@ import DeployService from '../deploy'; import { DeployStatus, DeployTypes } from 'shared/constants'; import { ChartType } from 'server/lib/nativeHelm'; import * as github from 'server/lib/github'; +import { SecretProcessor } from 'server/services/secretProcessor'; mockRedisClient(); @@ -31,6 +32,7 @@ const mockCodefreshWaitForImage = jest.fn(); const mockBuildWithNative = jest.fn(); const mockGlobalConfigGetAllConfigs = jest.fn(); const mockGlobalConfigGetOrgChartName = jest.fn(); +const mockEnsureNamespaceExists = jest.fn(); jest.mock('server/lib/logger', () => ({ getLogger: jest.fn(() => ({ @@ -67,6 +69,10 @@ jest.mock('server/lib/nativeBuild', () => ({ buildWithNative: (...args: any[]) => mockBuildWithNative(...args), })); +jest.mock('server/lib/nativeBuild/utils', () => ({ + ensureNamespaceExists: (...args: any[]) => mockEnsureNamespaceExists(...args), +})); + const mockDetermineChartType = jest.fn(); jest.mock('server/lib/nativeHelm', () => ({ ...jest.requireActual('server/lib/nativeHelm'), @@ -118,6 +124,7 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { mockCodefreshTagExists.mockReset(); mockCodefreshWaitForImage.mockReset(); mockBuildWithNative.mockReset(); + mockEnsureNamespaceExists.mockReset(); mockGlobalConfigGetOrgChartName.mockResolvedValue('org-chart'); mockGlobalConfigGetAllConfigs.mockResolvedValue({ lifecycleDefaults: { @@ -461,6 +468,269 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(patchSpy).toHaveBeenLastCalledWith(deploy, { status: DeployStatus.BUILD_FAILED }, 'run-1'); }); + test('buildImageForHelmAndGithub syncs external secrets when native image tag already exists', async () => { + (github.getSHAForBranch as jest.Mock).mockResolvedValue('abcdef1234567890'); + mockCodefreshTagExists.mockResolvedValue(true); + mockCodefreshGetRepositoryTag.mockReturnValue( + '123456789012.dkr.ecr.us-west-2.amazonaws.com/sample/app-images:lfc-abcdef1' + ); + mockGlobalConfigGetAllConfigs.mockResolvedValue({ + lifecycleDefaults: { + buildPipeline: 'sample/build-image', + deployCluster: 'test-cluster', + ecrDomain: '123456789012.dkr.ecr.us-west-2.amazonaws.com', + ecrRegistry: 'sample-registry', + }, + app_setup: { + org: 'example-org', + }, + buildDefaults: {}, + secretProviders: { + aws: { + enabled: true, + clusterSecretStore: 'aws-secretsmanager', + refreshInterval: '1h', + allowedPrefixes: [], + }, + }, + }); + + const processSecretsSpy = jest.spyOn(SecretProcessor.prototype, 'processEnvSecrets').mockResolvedValue({ + secretRefs: [ + { + envKey: 'API_TOKEN', + provider: 'aws', + path: 'repo/example-repo/api', + key: 'API_TOKEN', + }, + ], + expectedKeysPerSecret: { + 'sample-service-aws-secrets': ['API_TOKEN'], + }, + syncTokensPerSecret: { + 'sample-service-aws-secrets': 'sync-token', + }, + warnings: [], + }); + const waitForSecretSyncSpy = jest + .spyOn(SecretProcessor.prototype, 'waitForSecretSync') + .mockResolvedValue(undefined); + + const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); + const deployPatch = jest.fn().mockResolvedValue(undefined); + const deploy = { + uuid: 'sample-service-build', + runUUID: 'run-1', + branchName: 'feature-branch', + env: { + NODE_ENV: 'production', + API_TOKEN: '{{aws:repo/example-repo/api:API_TOKEN}}', + }, + initEnv: {}, + dockerImage: 'old-image', + service: { + name: 'sample-service', + }, + build: { + id: 1, + uuid: 'sample-build', + namespace: 'env-sample', + enableFullYaml: true, + commentRuntimeEnv: {}, + enabledFeatures: [], + pullRequest: { + githubLogin: 'sample-user', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + deployable: { + name: 'sample-service', + type: DeployTypes.GITHUB, + dockerfilePath: './Dockerfile', + initDockerfilePath: null, + env: {}, + ecr: 'sample/app-images', + dockerBuildPipelineName: 'sample/build-image', + builder: { + engine: 'buildkit', + }, + repository: { + fullName: 'example-org/example-repo', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + $query: jest.fn(() => ({ + patch: deployPatch, + })), + }; + + try { + const result = await deployService.buildImageForHelmAndGithub(deploy as any, 'run-1'); + + expect(result).toBe(true); + expect(mockBuildWithNative).not.toHaveBeenCalled(); + expect(mockEnsureNamespaceExists).toHaveBeenCalledWith('env-sample'); + expect(processSecretsSpy).toHaveBeenCalledWith({ + env: { + API_TOKEN: '{{aws:repo/example-repo/api:API_TOKEN}}', + }, + serviceName: 'sample-service', + namespace: 'env-sample', + buildUuid: 'sample-service-build', + }); + expect(waitForSecretSyncSpy).toHaveBeenCalledWith( + { + 'sample-service-aws-secrets': ['API_TOKEN'], + }, + 'env-sample', + 60000, + { + 'sample-service-aws-secrets': 'sync-token', + } + ); + expect(deployPatch).toHaveBeenCalledWith( + expect.objectContaining({ + status: DeployStatus.BUILT, + dockerImage: '123456789012.dkr.ecr.us-west-2.amazonaws.com/sample/app-images:lfc-abcdef1', + }) + ); + expect(patchSpy).toHaveBeenLastCalledWith(deploy, { status: DeployStatus.BUILT }, 'run-1'); + } finally { + processSecretsSpy.mockRestore(); + waitForSecretSyncSpy.mockRestore(); + } + }); + + test('buildImageForHelmAndGithub syncs comment init env secrets when native image tag already exists', async () => { + (github.getSHAForBranch as jest.Mock).mockResolvedValue('abcdef1234567890'); + mockCodefreshTagExists.mockResolvedValue(true); + mockCodefreshGetRepositoryTag.mockReturnValue( + '123456789012.dkr.ecr.us-west-2.amazonaws.com/sample/app-images:lfc-abcdef1' + ); + mockGlobalConfigGetAllConfigs.mockResolvedValue({ + lifecycleDefaults: { + buildPipeline: 'sample/build-image', + deployCluster: 'test-cluster', + ecrDomain: '123456789012.dkr.ecr.us-west-2.amazonaws.com', + ecrRegistry: 'sample-registry', + }, + app_setup: { + org: 'example-org', + }, + buildDefaults: {}, + secretProviders: { + aws: { + enabled: true, + clusterSecretStore: 'aws-secretsmanager', + refreshInterval: '1h', + allowedPrefixes: [], + }, + }, + }); + + const processSecretsSpy = jest.spyOn(SecretProcessor.prototype, 'processEnvSecrets').mockResolvedValue({ + secretRefs: [ + { + envKey: 'INIT_TOKEN', + provider: 'aws', + path: 'repo/example-repo/api', + key: 'INIT_TOKEN', + }, + ], + expectedKeysPerSecret: { + 'sample-service-aws-secrets': ['INIT_TOKEN'], + }, + syncTokensPerSecret: { + 'sample-service-aws-secrets': 'sync-token', + }, + warnings: [], + }); + const waitForSecretSyncSpy = jest + .spyOn(SecretProcessor.prototype, 'waitForSecretSync') + .mockResolvedValue(undefined); + + const deployPatch = jest.fn().mockResolvedValue(undefined); + const deploy = { + uuid: 'sample-service-build', + runUUID: 'run-1', + branchName: 'feature-branch', + env: { + NODE_ENV: 'production', + }, + initEnv: {}, + dockerImage: 'old-image', + service: { + name: 'sample-service', + }, + build: { + id: 1, + uuid: 'sample-build', + namespace: 'env-sample', + enableFullYaml: true, + commentRuntimeEnv: {}, + commentInitEnv: { + INIT_TOKEN: '{{aws:repo/example-repo/api:INIT_TOKEN}}', + }, + enabledFeatures: [], + pullRequest: { + githubLogin: 'sample-user', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + deployable: { + name: 'sample-service', + type: DeployTypes.GITHUB, + dockerfilePath: './Dockerfile', + initDockerfilePath: './init.Dockerfile', + env: {}, + ecr: 'sample/app-images', + dockerBuildPipelineName: 'sample/build-image', + builder: { + engine: 'buildkit', + }, + repository: { + fullName: 'example-org/example-repo', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + $query: jest.fn(() => ({ + patch: deployPatch, + })), + }; + + try { + const result = await deployService.buildImageForHelmAndGithub(deploy as any, 'run-1'); + + expect(result).toBe(true); + expect(mockBuildWithNative).not.toHaveBeenCalled(); + expect(processSecretsSpy).toHaveBeenCalledWith({ + env: { + INIT_TOKEN: '{{aws:repo/example-repo/api:INIT_TOKEN}}', + }, + serviceName: 'sample-service', + namespace: 'env-sample', + buildUuid: 'sample-service-build', + }); + expect(waitForSecretSyncSpy).toHaveBeenCalledWith( + { + 'sample-service-aws-secrets': ['INIT_TOKEN'], + }, + 'env-sample', + 60000, + { + 'sample-service-aws-secrets': 'sync-token', + } + ); + } finally { + processSecretsSpy.mockRestore(); + waitForSecretSyncSpy.mockRestore(); + } + }); + test('deployAurora records failures with the newly assigned runUUID', async () => { const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); jest.spyOn(deployService as any, 'findExistingAuroraDatabase').mockResolvedValue(null); diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index db1ad223..24a0a911 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -41,6 +41,7 @@ import { parseSecretRefsFromEnv } from 'server/lib/secretRefs'; import { SecretProcessor } from 'server/services/secretProcessor'; import { fallbackDeployStatusMessage, statusMessageFromError } from 'server/lib/terminalFailure'; import { isNativeBuilderEngine } from 'server/lib/buildEngines'; +import { SecretProvidersConfig } from 'server/services/types/globalConfig'; export interface DeployOptions { ownerId?: number; @@ -60,6 +61,11 @@ export interface PipelineWaitItem { patternInfo: PatternInfo[]; } +interface SyncedServiceExternalSecrets { + secretNames: string[]; + buildSecretEnvKeys: Set; +} + export default class DeployService extends BaseService { /** * Creates all of the relevant deploys for a build, based on the provided environment, if they do not already exist. @@ -969,6 +975,116 @@ export default class DeployService extends BaseService { }); } + private async syncServiceExternalSecrets({ + deploy, + serviceName, + secretProviders, + runUUID, + }: { + deploy: Deploy; + serviceName: string; + secretProviders: SecretProvidersConfig | undefined; + runUUID: string; + }): Promise { + const emptyResult = { secretNames: [], buildSecretEnvKeys: new Set() }; + + if (!secretProviders) { + return emptyResult; + } + + const buildEnvToProcess = merge({}, deploy.env || {}, deploy.build?.commentRuntimeEnv || {}) as Record< + string, + string + >; + const initEnvToProcess = merge({}, deploy.initEnv || {}, deploy.build?.commentInitEnv || {}) as Record< + string, + string + >; + const buildSecretRefs = parseSecretRefsFromEnv(buildEnvToProcess); + const initSecretRefs = parseSecretRefsFromEnv(initEnvToProcess); + const buildSecretRefKeys = new Set(buildSecretRefs.map((ref) => ref.envKey)); + const combinedSecretEnvEntries = new Map(); + const conflictingSecretEnvKeys = new Set(); + + const addSecretRef = (envKey: string, value: string) => { + const existingValue = combinedSecretEnvEntries.get(envKey); + + if (existingValue && existingValue !== value) { + conflictingSecretEnvKeys.add(envKey); + return; + } + + combinedSecretEnvEntries.set(envKey, value); + }; + + buildSecretRefs.forEach((ref) => addSecretRef(ref.envKey, buildEnvToProcess[ref.envKey])); + initSecretRefs.forEach((ref) => addSecretRef(ref.envKey, initEnvToProcess[ref.envKey])); + + if (conflictingSecretEnvKeys.size > 0) { + getLogger().error( + `Build: secret env conflict service=${serviceName} keys=[${Array.from(conflictingSecretEnvKeys).join(', ')}]` + ); + await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); + return false; + } + + const envToProcess = Object.fromEntries(combinedSecretEnvEntries); + + if (Object.keys(envToProcess).length === 0) { + return emptyResult; + } + + const { ensureNamespaceExists } = await import('server/lib/nativeBuild/utils'); + await ensureNamespaceExists(deploy.build.namespace); + + const secretProcessor = new SecretProcessor(secretProviders); + + const secretResult = await secretProcessor.processEnvSecrets({ + env: envToProcess, + serviceName, + namespace: deploy.build.namespace, + buildUuid: deploy.uuid, + }); + + const buildSecretEnvKeys = new Set( + secretResult.secretRefs.filter((ref) => buildSecretRefKeys.has(ref.envKey)).map((ref) => ref.envKey) + ); + + if (secretResult.warnings.length > 0) { + getLogger().warn( + `Build: secret processing warnings service=${serviceName} warnings=${secretResult.warnings.join(', ')}` + ); + } + + const secretNames = Object.keys(secretResult.expectedKeysPerSecret); + + if (secretNames.length === 0) { + return { secretNames, buildSecretEnvKeys }; + } + + getLogger().info(`Build: waiting for secrets to sync secrets=[${secretNames.join(', ')}]`); + + const providerTimeouts = Object.values(secretProviders) + .map((p) => p.secretSyncTimeout) + .filter((t): t is number => t !== undefined); + const timeout = providerTimeouts.length > 0 ? Math.max(...providerTimeouts) * 1000 : 60000; + + try { + await secretProcessor.waitForSecretSync( + secretResult.expectedKeysPerSecret, + deploy.build.namespace, + timeout, + secretResult.syncTokensPerSecret + ); + getLogger().info(`Build: secrets synced count=${secretNames.length}`); + return { secretNames, buildSecretEnvKeys }; + } catch (error) { + getLogger().error({ error }, `Build: secret sync failed service=${serviceName}`); + await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); + return false; + } + } + hostForServiceDeploy(deploy: Deploy, service: Service) { if (service.type === DeployTypes.EXTERNAL_HTTP) { return deploy.publicUrl ? deploy.publicUrl : service.defaultPublicUrl; @@ -1031,7 +1147,8 @@ export default class DeployService extends BaseService { const initTag = generateDeployTag({ prefix: 'lfc-init', sha: shortSha, envVarsHash }); let ecrRepo = deployable?.ecr; - const { lifecycleDefaults, app_setup, buildDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); + const { lifecycleDefaults, app_setup, buildDefaults, secretProviders } = + await GlobalConfigService.getInstance().getAllConfigs(); const { ecrDomain, ecrRegistry: registry } = lifecycleDefaults; const serviceName = deploy.build?.enableFullYaml ? deployable?.name : deploy.service?.name; @@ -1092,103 +1209,23 @@ export default class DeployService extends BaseService { if (isNativeBuilderEngine(deployable.builder?.engine)) { getLogger().info(`Image: building engine=${deployable.builder.engine}`); - let buildSecretNames: string[] = []; - let secretEnvKeys: Set = new Set(); - const globalConfigs = await GlobalConfigService.getInstance().getAllConfigs(); - const secretProviders = globalConfigs.secretProviders; - - if (secretProviders) { - const buildEnvToProcess = (deploy.env || {}) as Record; - const initEnvToProcess = (deploy.initEnv || {}) as Record; - const buildSecretRefs = parseSecretRefsFromEnv(buildEnvToProcess); - const initSecretRefs = parseSecretRefsFromEnv(initEnvToProcess); - const buildSecretRefKeys = new Set(buildSecretRefs.map((ref) => ref.envKey)); - const combinedSecretEnvEntries = new Map(); - const conflictingSecretEnvKeys = new Set(); - - const addSecretRef = (envKey: string, value: string) => { - const existingValue = combinedSecretEnvEntries.get(envKey); - - if (existingValue && existingValue !== value) { - conflictingSecretEnvKeys.add(envKey); - return; - } - - combinedSecretEnvEntries.set(envKey, value); - }; - - buildSecretRefs.forEach((ref) => addSecretRef(ref.envKey, buildEnvToProcess[ref.envKey])); - initSecretRefs.forEach((ref) => addSecretRef(ref.envKey, initEnvToProcess[ref.envKey])); - - if (conflictingSecretEnvKeys.size > 0) { - getLogger().error( - `Build: secret env conflict service=${deployable.name} keys=[${Array.from( - conflictingSecretEnvKeys - ).join(', ')}]` - ); - await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); - return false; - } - - const envToProcess = Object.fromEntries(combinedSecretEnvEntries); - - if (Object.keys(envToProcess).length > 0) { - const { ensureNamespaceExists } = await import('server/lib/nativeBuild/utils'); - await ensureNamespaceExists(deploy.build.namespace); - - const secretProcessor = new SecretProcessor(secretProviders); - - const secretResult = await secretProcessor.processEnvSecrets({ - env: envToProcess, - serviceName: deployable.name, - namespace: deploy.build.namespace, - buildUuid: deploy.uuid, - }); - - secretEnvKeys = new Set( - secretResult.secretRefs.filter((ref) => buildSecretRefKeys.has(ref.envKey)).map((ref) => ref.envKey) - ); - - if (secretResult.warnings.length > 0) { - getLogger().warn( - `Build: secret processing warnings service=${deployable.name} warnings=${secretResult.warnings.join( - ', ' - )}` - ); - } - - const secretNames = Object.keys(secretResult.expectedKeysPerSecret); - - if (secretNames.length > 0) { - getLogger().info(`Build: waiting for secrets to sync secrets=[${secretNames.join(', ')}]`); - - const providerTimeouts = Object.values(secretProviders) - .map((p) => p.secretSyncTimeout) - .filter((t): t is number => t !== undefined); - const timeout = providerTimeouts.length > 0 ? Math.max(...providerTimeouts) * 1000 : 60000; + const syncedExternalSecrets = await this.syncServiceExternalSecrets({ + deploy, + serviceName: deployable.name, + secretProviders, + runUUID, + }); - try { - await secretProcessor.waitForSecretSync( - secretResult.expectedKeysPerSecret, - deploy.build.namespace, - timeout, - secretResult.syncTokensPerSecret - ); - buildSecretNames = secretNames; - getLogger().info(`Build: secrets synced count=${buildSecretNames.length}`); - } catch (error) { - getLogger().error({ error }, `Build: secret sync failed service=${deployable.name}`); - await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILD_FAILED }, runUUID); - return false; - } - } - } + if (!syncedExternalSecrets) { + return false; } const filteredEnvVars = - secretEnvKeys.size > 0 + syncedExternalSecrets.buildSecretEnvKeys.size > 0 ? Object.fromEntries( - Object.entries(buildOptions.envVars || {}).filter(([key]) => !secretEnvKeys.has(key)) + Object.entries(buildOptions.envVars || {}).filter( + ([key]) => !syncedExternalSecrets.buildSecretEnvKeys.has(key) + ) ) : buildOptions.envVars; @@ -1202,8 +1239,8 @@ export default class DeployService extends BaseService { cacheRegistry: buildDefaults?.cacheRegistry, resources: deployable.builder?.resources, podAnnotations: deployable.builder?.podAnnotations, - secretRefs: buildSecretNames, - secretEnvKeys: Array.from(secretEnvKeys), + secretRefs: syncedExternalSecrets.secretNames, + secretEnvKeys: Array.from(syncedExternalSecrets.buildSecretEnvKeys), }; if (!initDockerfilePath) { @@ -1253,6 +1290,18 @@ export default class DeployService extends BaseService { } } else { getLogger().info('Image: skipped reason=exists'); + if (isNativeBuilderEngine(deployable.builder?.engine)) { + const syncedExternalSecrets = await this.syncServiceExternalSecrets({ + deploy, + serviceName: deployable.name, + secretProviders, + runUUID, + }); + + if (!syncedExternalSecrets) { + return false; + } + } await this.patchDeployWithTag({ tag, initTag, deploy, ecrDomain }); await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.BUILT }, runUUID); return true;