diff --git a/docs/schema/yaml/1.0.0.yaml b/docs/schema/yaml/1.0.0.yaml index 89061fed..64772fc7 100644 --- a/docs/schema/yaml/1.0.0.yaml +++ b/docs/schema/yaml/1.0.0.yaml @@ -15,8 +15,7 @@ environment: # @param environment.defaultServices defaultServices: # @param environment.defaultServices[] - - - # @param environment.defaultServices.name (required) + - # @param environment.defaultServices.name (required) name: '' # @param environment.defaultServices.repository repository: '' @@ -27,8 +26,7 @@ environment: # @param environment.optionalServices optionalServices: # @param environment.optionalServices[] - - - # @param environment.optionalServices.name (required) + - # @param environment.optionalServices.name (required) name: '' # @param environment.optionalServices.repository repository: '' @@ -39,8 +37,7 @@ environment: # @param environment.webhooks webhooks: # @param environment.webhooks[] - - - # @param environment.webhooks.name + - # @param environment.webhooks.name name: '' # @param environment.webhooks.description description: '' @@ -50,11 +47,8 @@ environment: type: '' # @param environment.webhooks.pipelineId pipelineId: '' - # @param environment.webhooks.trigger (required) + # @param environment.webhooks.trigger trigger: '' - # @param environment.webhooks.env (required) - env: - # @param environment.webhooks.docker docker: # @param environment.webhooks.docker.image (required) @@ -77,11 +71,13 @@ environment: script: '' # @param environment.webhooks.command.timeout timeout: 0 + # @param environment.webhooks.env (required) + env: + # @section services services: # @param services[] - - - # @param services.name (required) + - # @param services.name (required) name: '' # @param services.appShort appShort: '' @@ -90,9 +86,40 @@ services: # @param services.requires requires: # @param services.requires[] - - - # @param services.requires.name (required) + - # @param services.requires.name (required) name: '' + # @param services.deploymentDependsOn + deploymentDependsOn: + # @param services.deploymentDependsOn[] + - '' + # @param services.kedaScaleToZero + kedaScaleToZero: + # @param services.kedaScaleToZero.type + type: '' + # @param services.kedaScaleToZero.replicas + replicas: + # @param services.kedaScaleToZero.replicas.min + min: 0 + # @param services.kedaScaleToZero.replicas.max + max: 0 + # @param services.kedaScaleToZero.scaledownPeriod + scaledownPeriod: 0 + # @param services.kedaScaleToZero.maxRetries + maxRetries: 0 + # @param services.kedaScaleToZero.scalingMetric + scalingMetric: + # @param services.kedaScaleToZero.scalingMetric.requestRate + requestRate: + # @param services.kedaScaleToZero.scalingMetric.requestRate.granularity + granularity: '' + # @param services.kedaScaleToZero.scalingMetric.requestRate.targetValue + targetValue: 0 + # @param services.kedaScaleToZero.scalingMetric.requestRate.window + window: '' + # @param services.kedaScaleToZero.scalingMetric.concurrency + concurrency: + # @param services.kedaScaleToZero.scalingMetric.concurrency.targetValue + targetValue: 0 # @param services.helm helm: # @param services.helm.cfStepType @@ -105,9 +132,9 @@ services: version: '' # @param services.helm.action action: '' - # @param services.helm.repository (required) + # @param services.helm.repository repository: '' - # @param services.helm.branchName (required) + # @param services.helm.branchName branchName: '' # @param services.helm.chart chart: @@ -125,6 +152,8 @@ services: valueFiles: # @param services.helm.chart.valueFiles[] - '' + # @param services.helm.envLens + envLens: false # @param services.helm.grpc grpc: false # @param services.helm.disableIngressHost @@ -137,6 +166,12 @@ services: defaultTag: '' # @param services.helm.docker.pipelineId pipelineId: '' + # @param services.helm.docker.ecr + ecr: '' + # @param services.helm.docker.builder + builder: + # @param services.helm.docker.builder.engine + engine: '' # @param services.helm.docker.app (required) app: # @param services.helm.docker.app.afterBuildPipelineConfig @@ -171,14 +206,6 @@ services: # @param services.helm.docker.init.env env: - # @param services.helm.docker.ecr - ecr: '' - # @param services.helm.docker.builder - builder: - # @param services.helm.docker.builder.engine - engine: '' - # @param services.helm.envLens - envLens: false # @param services.codefresh codefresh: # @param services.codefresh.repository (required) @@ -272,12 +299,6 @@ services: successThreshold: 0 # @param services.codefresh.deployment.readiness.failureThreshold failureThreshold: 0 - # @param services.codefresh.deployment.scaling - scaling: - # @param services.codefresh.deployment.scaling.scaleToZero - scaleToZero: false - # @param services.codefresh.deployment.scaling.scaleToZeroMetricsCheckInterval - scaleToZeroMetricsCheckInterval: 0 # @param services.codefresh.deployment.hostnames hostnames: # @param services.codefresh.deployment.hostnames.host @@ -300,9 +321,6 @@ services: # @param services.codefresh.deployment.network.hostPortMapping hostPortMapping: - # @param services.codefresh.deployment.network.kuma - kuma: - # @param services.codefresh.deployment.network.grpc grpc: # @param services.codefresh.deployment.network.grpc.enable @@ -314,8 +332,7 @@ services: # @param services.codefresh.deployment.serviceDisks serviceDisks: # @param services.codefresh.deployment.serviceDisks[] - - - # @param services.codefresh.deployment.serviceDisks.name (required) + - # @param services.codefresh.deployment.serviceDisks.name (required) name: '' # @param services.codefresh.deployment.serviceDisks.mountPath (required) mountPath: '' @@ -337,6 +354,12 @@ services: defaultTag: '' # @param services.github.docker.pipelineId pipelineId: '' + # @param services.github.docker.ecr + ecr: '' + # @param services.github.docker.builder + builder: + # @param services.github.docker.builder.engine + engine: '' # @param services.github.docker.app (required) app: # @param services.github.docker.app.afterBuildPipelineConfig @@ -371,12 +394,6 @@ services: # @param services.github.docker.init.env env: - # @param services.github.docker.ecr - ecr: '' - # @param services.github.docker.builder - builder: - # @param services.github.docker.builder.engine - engine: '' # @param services.github.deployment deployment: # @param services.github.deployment.helm @@ -449,12 +466,6 @@ services: successThreshold: 0 # @param services.github.deployment.readiness.failureThreshold failureThreshold: 0 - # @param services.github.deployment.scaling - scaling: - # @param services.github.deployment.scaling.scaleToZero - scaleToZero: false - # @param services.github.deployment.scaling.scaleToZeroMetricsCheckInterval - scaleToZeroMetricsCheckInterval: 0 # @param services.github.deployment.hostnames hostnames: # @param services.github.deployment.hostnames.host @@ -477,9 +488,6 @@ services: # @param services.github.deployment.network.hostPortMapping hostPortMapping: - # @param services.github.deployment.network.kuma - kuma: - # @param services.github.deployment.network.grpc grpc: # @param services.github.deployment.network.grpc.enable @@ -491,8 +499,7 @@ services: # @param services.github.deployment.serviceDisks serviceDisks: # @param services.github.deployment.serviceDisks[] - - - # @param services.github.deployment.serviceDisks.name (required) + - # @param services.github.deployment.serviceDisks.name (required) name: '' # @param services.github.deployment.serviceDisks.mountPath (required) mountPath: '' @@ -591,12 +598,6 @@ services: successThreshold: 0 # @param services.docker.deployment.readiness.failureThreshold failureThreshold: 0 - # @param services.docker.deployment.scaling - scaling: - # @param services.docker.deployment.scaling.scaleToZero - scaleToZero: false - # @param services.docker.deployment.scaling.scaleToZeroMetricsCheckInterval - scaleToZeroMetricsCheckInterval: 0 # @param services.docker.deployment.hostnames hostnames: # @param services.docker.deployment.hostnames.host @@ -619,9 +620,6 @@ services: # @param services.docker.deployment.network.hostPortMapping hostPortMapping: - # @param services.docker.deployment.network.kuma - kuma: - # @param services.docker.deployment.network.grpc grpc: # @param services.docker.deployment.network.grpc.enable @@ -633,8 +631,7 @@ services: # @param services.docker.deployment.serviceDisks serviceDisks: # @param services.docker.deployment.serviceDisks[] - - - # @param services.docker.deployment.serviceDisks.name (required) + - # @param services.docker.deployment.serviceDisks.name (required) name: '' # @param services.docker.deployment.serviceDisks.mountPath (required) mountPath: '' @@ -662,35 +659,3 @@ services: defaultTag: '' # @param services.configuration.branchName (required) branchName: '' - # @param services.deploymentDependsOn - deploymentDependsOn: - # @param services.deploymentDependsOn[] - - '' - # @param services.kedaScaleToZero - kedaScaleToZero: - # @param services.kedaScaleToZero.type - type: '' - # @param services.kedaScaleToZero.replicas - replicas: - # @param services.kedaScaleToZero.replicas.min - min: 0 - # @param services.kedaScaleToZero.replicas.max - max: 0 - # @param services.kedaScaleToZero.scaledownPeriod - scaledownPeriod: 0 - # @param services.kedaScaleToZero.maxRetries - maxRetries: 0 - # @param services.kedaScaleToZero.scalingMetric - scalingMetric: - # @param services.kedaScaleToZero.scalingMetric.requestRate - requestRate: - # @param services.kedaScaleToZero.scalingMetric.requestRate.granularity - granularity: '' - # @param services.kedaScaleToZero.scalingMetric.requestRate.targetValue - targetValue: 0 - # @param services.kedaScaleToZero.scalingMetric.requestRate.window - window: '' - # @param services.kedaScaleToZero.scalingMetric.concurrency - concurrency: - # @param services.kedaScaleToZero.scalingMetric.concurrency.targetValue - targetValue: 0 \ No newline at end of file diff --git a/src/server/db/migrations/001_seed.ts b/src/server/db/migrations/001_seed.ts index 8bdf3a22..4de01d8f 100644 --- a/src/server/db/migrations/001_seed.ts +++ b/src/server/db/migrations/001_seed.ts @@ -130,11 +130,11 @@ export async function up(knex: Knex): Promise { ); ALTER TABLE builds OWNER TO lifecycle; INSERT INTO builds ( - id, uuid, status, "statusMessage", "environmentId", + uuid, status, "statusMessage", "environmentId", "createdAt", "updatedAt", sha, namespace ) VALUES ( - 99999, 'dev-0', 'pending', 'Build is pending', 1, + 'dev-0', 'pending', 'Build is pending', 1, NOW(), NOW(), 'abc123', 'env-dev-0' ) `); diff --git a/src/server/lib/cli.ts b/src/server/lib/cli.ts index 7811adf0..25ebafd3 100644 --- a/src/server/lib/cli.ts +++ b/src/server/lib/cli.ts @@ -54,8 +54,10 @@ export async function cliDeploy(deploy: Deploy) { const { build, service, deployable } = deploy; const serviceCommand: string = build.enableFullYaml ? deployable.command : service.command; - const settings = await getSettingsFor(serviceCommand); - return await shellPromise(`pnpm run babel-node -- ${serviceCommand} deploy ${contextForDeploy(deploy, settings)}`); + const { settings, region } = await getSettingsFor(serviceCommand); + return await shellPromise( + `AWS_REGION=${region} pnpm run babel-node -- ${serviceCommand} deploy ${contextForDeploy(deploy, settings)}` + ); } /** @@ -214,11 +216,13 @@ function contextForDeploy(deploy: Deploy, settings: string) { async function deleteDeploy(deploy: Deploy) { const serviceCmd = deploy.build.enableFullYaml ? deploy.deployable.command : deploy.service.command; - const settings = await getSettingsFor(serviceCmd); - return await shellPromise(`pnpm run babel-node -- ${serviceCmd} destroy ${contextForDeploy(deploy, settings)}`); + const { settings, region } = await getSettingsFor(serviceCmd); + return await shellPromise( + `AWS_REGION=${region} pnpm run babel-node -- ${serviceCmd} destroy ${contextForDeploy(deploy, settings)}` + ); } -async function getSettingsFor(serviceCommand: string): Promise { +async function getSettingsFor(serviceCommand: string): Promise<{ settings: string; region: string }> { const { auroraRestoreSettings, rdsRestoreSettings } = await GlobalConfigService.getInstance().getAllConfigs(); let settings: DatabaseSettings; if (serviceCommand.includes('aurora-helper')) { @@ -226,5 +230,8 @@ async function getSettingsFor(serviceCommand: string): Promise { } else if (serviceCommand.includes('rds-helper')) { settings = rdsRestoreSettings; } - return JSON.stringify(settings); + return { + settings: JSON.stringify(settings), + region: settings.region, + }; } diff --git a/src/server/lib/deploymentManager/deploymentManager.ts b/src/server/lib/deploymentManager/deploymentManager.ts index 4d36e327..d5b0956f 100644 --- a/src/server/lib/deploymentManager/deploymentManager.ts +++ b/src/server/lib/deploymentManager/deploymentManager.ts @@ -22,6 +22,7 @@ import { nanoid, customAlphabet } from 'nanoid'; import DeployService from 'server/services/deploy'; import rootLogger from 'server/lib/logger'; import { ensureServiceAccountForJob } from '../kubernetes/common/serviceAccount'; +import { waitForDeployPodReady } from '../kubernetes'; const logger = rootLogger.child({ filename: 'lib/deploymentManager/deploymentManager.ts' }); const generateJobId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 6); @@ -121,7 +122,7 @@ export class DeploymentManager { await Promise.all([ helmDeploys.length > 0 ? deployHelm(helmDeploys) : Promise.resolve(), - ...githubDeploys.map((deploy) => this.deployGitHubDeploy(deploy)), + ...githubDeploys.map((deploy) => this.deployManifests(deploy)), ]); } } @@ -134,10 +135,11 @@ export class DeploymentManager { private shouldDeployWithKubernetes(deploy: Deploy): boolean { const deployType = deploy.deployable?.type || deploy.service?.type; - return deployType === DeployTypes.GITHUB || deployType === DeployTypes.DOCKER || CLIDeployTypes.has(deployType); + // Note: only the below types have Kubernetes manifests + return [DeployTypes.GITHUB, DeployTypes.DOCKER, DeployTypes.AURORA_RESTORE].includes(deployType); } - private async deployGitHubDeploy(deploy: Deploy): Promise { + private async deployManifests(deploy: Deploy): Promise { const jobId = generateJobId(); const deployService = new DeployService(); const runUUID = deploy.runUUID || nanoid(); @@ -170,34 +172,33 @@ export class DeploymentManager { const jobName = `${deploy.uuid}-deploy-${jobId}-${shortSha}`; const result = await monitorKubernetesJob(jobName, deploy.build.namespace); - if (result.success) { - // Wait for the actual application pods to be ready + if (!result.success) { + throw new Error(result.message); + } + // Wait for the actual application pods to be ready + await deployService.patchAndUpdateActivityFeed( + deploy, + { + status: DeployStatus.DEPLOYING, + statusMessage: 'Waiting for pods to be ready', + }, + runUUID + ); + + const cliDeploy = CLIDeployTypes.has(deploy.deployable.type); + const isReady = cliDeploy ? true : await waitForDeployPodReady(deploy); + + if (isReady) { await deployService.patchAndUpdateActivityFeed( deploy, { - status: DeployStatus.DEPLOYING, - statusMessage: 'Waiting for pods to be ready', + status: DeployStatus.READY, + statusMessage: cliDeploy ? 'CLI Deploy completed' : 'Kubernetes pods are ready', }, runUUID ); - - const { waitForDeployPodReady } = await import('../kubernetes'); - const isReady = await waitForDeployPodReady(deploy); - - if (isReady) { - await deployService.patchAndUpdateActivityFeed( - deploy, - { - status: DeployStatus.READY, - statusMessage: 'Kubernetes pods are ready', - }, - runUUID - ); - } else { - throw new Error('Pods failed to become ready within timeout'); - } } else { - throw new Error(result.message); + throw new Error('Pods failed to become ready within timeout'); } } catch (error) { await deployService.patchAndUpdateActivityFeed( diff --git a/src/server/lib/helm/__tests__/helm.test.ts b/src/server/lib/helm/__tests__/helm.test.ts index 084dd9b4..91ff0d3d 100644 --- a/src/server/lib/helm/__tests__/helm.test.ts +++ b/src/server/lib/helm/__tests__/helm.test.ts @@ -17,13 +17,16 @@ import mockRedisClient from 'server/lib/__mocks__/redisClientMock'; mockRedisClient(); -import { constructHelmDeploysBuildMetaData } from 'server/lib/helm'; +import { constructHelmDeploysBuildMetaData, grpcMapping } from 'server/lib/helm'; import { Deploy } from 'server/models'; +import GlobalConfigService from 'server/services/globalConfig'; jest.mock('server/lib/envVariables', () => ({ EnvironmentVariables: class {}, })); +jest.mock('server/services/globalConfig'); + describe('Helm tests', () => { test('constructHelmDeploysBuildMetaData should return the correct metadata', async () => { const deploys = [ @@ -61,4 +64,95 @@ describe('Helm tests', () => { const metadata = await constructHelmDeploysBuildMetaData(deploys); expect(metadata).toEqual({ branchName: '', fullName: '', sha: '', uuid: '', error: 'no_related_build_found' }); }); + + describe('grpcMapping', () => { + const mockDeploy = { + uuid: 'test-deploy-uuid', + deployable: { + buildUUID: 'test-build-uuid', + port: 8080, + }, + } as unknown as Deploy; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('should create single ambassador mapping when altGrpc is empty', async () => { + const mockGetAllConfigs = jest.fn().mockResolvedValue({ + domainDefaults: { + grpc: 'grpc.example.com', + altGrpc: [], + }, + }); + + (GlobalConfigService.getInstance as jest.Mock).mockReturnValue({ + getAllConfigs: mockGetAllConfigs, + }); + + const result = await grpcMapping(mockDeploy); + + expect(result).toEqual([ + 'ambassadorMappings[0].name=test-deploy-uuid-0', + 'ambassadorMappings[0].env=lifecycle-test-build-uuid', + 'ambassadorMappings[0].service=test-deploy-uuid', + 'ambassadorMappings[0].version=test-deploy-uuid', + 'ambassadorMappings[0].host=test-deploy-uuid.grpc.example.com:443', + 'ambassadorMappings[0].port=8080', + ]); + }); + + test('should create single ambassador mapping when altGrpc is undefined', async () => { + const mockGetAllConfigs = jest.fn().mockResolvedValue({ + domainDefaults: { + grpc: 'grpc.example.com', + }, + }); + + (GlobalConfigService.getInstance as jest.Mock).mockReturnValue({ + getAllConfigs: mockGetAllConfigs, + }); + + const result = await grpcMapping(mockDeploy); + + expect(result).toEqual([ + 'ambassadorMappings[0].name=test-deploy-uuid-0', + 'ambassadorMappings[0].env=lifecycle-test-build-uuid', + 'ambassadorMappings[0].service=test-deploy-uuid', + 'ambassadorMappings[0].version=test-deploy-uuid', + 'ambassadorMappings[0].host=test-deploy-uuid.grpc.example.com:443', + 'ambassadorMappings[0].port=8080', + ]); + }); + + test('should create multiple ambassador mappings when altGrpc has values', async () => { + const mockGetAllConfigs = jest.fn().mockResolvedValue({ + domainDefaults: { + grpc: 'grpc.example.com', + altGrpc: ['grpc-alt.example.com'], + }, + }); + + (GlobalConfigService.getInstance as jest.Mock).mockReturnValue({ + getAllConfigs: mockGetAllConfigs, + }); + + const result = await grpcMapping(mockDeploy); + + expect(result).toEqual([ + 'ambassadorMappings[0].name=test-deploy-uuid-0', + 'ambassadorMappings[0].env=lifecycle-test-build-uuid', + 'ambassadorMappings[0].service=test-deploy-uuid', + 'ambassadorMappings[0].version=test-deploy-uuid', + 'ambassadorMappings[0].host=test-deploy-uuid.grpc.example.com:443', + 'ambassadorMappings[0].port=8080', + 'ambassadorMappings[1].name=test-deploy-uuid-1', + 'ambassadorMappings[1].env=lifecycle-test-build-uuid', + 'ambassadorMappings[1].service=test-deploy-uuid', + 'ambassadorMappings[1].version=test-deploy-uuid', + 'ambassadorMappings[1].host=test-deploy-uuid.grpc-alt.example.com:443', + 'ambassadorMappings[1].port=8080', + ]); + }); + }); }); diff --git a/src/server/lib/helm/helm.ts b/src/server/lib/helm/helm.ts index ba51a1ae..a8d96748 100644 --- a/src/server/lib/helm/helm.ts +++ b/src/server/lib/helm/helm.ts @@ -161,15 +161,8 @@ export async function helmOrgAppDeployStep(deploy: Deploy): Promise { + const { domainDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); + const hosts = [domainDefaults.grpc, ...(domainDefaults?.altGrpc || [])]; + + const mappings: string[] = []; + + hosts.forEach((host, index) => { + mappings.push( + `ambassadorMappings[${index}].name=${deploy.uuid}-${index}`, + `ambassadorMappings[${index}].env=lifecycle-${deploy.deployable.buildUUID}`, + `ambassadorMappings[${index}].service=${deploy.uuid}`, + `ambassadorMappings[${index}].version=${deploy.uuid}`, + `ambassadorMappings[${index}].host=${deploy.uuid}.${host}:443`, + `ambassadorMappings[${index}].port=${deploy.deployable.port}` + ); + }); + + return mappings; +} + async function httpIngress(deploy: Deploy): Promise { let ingressValues = []; const { serviceDefaults, domainDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); ingressValues = [`ingress.host=${deploy.uuid}.${domainDefaults.http}`]; + if (domainDefaults?.altHttp) { + domainDefaults.altHttp.forEach((host, index) => { + ingressValues.push(`ingress.altHosts[${index}]=${deploy.uuid}.${host}`); + }); + } if (!deploy.deployable.helm.overrideDefaultIpWhitelist) { const ipWhitelist = serviceDefaults.defaultIPWhiteList .trim() diff --git a/src/server/lib/jsonschema/schemas/1.0.0.json b/src/server/lib/jsonschema/schemas/1.0.0.json index 1ffdd8dd..0bfe794b 100644 --- a/src/server/lib/jsonschema/schemas/1.0.0.json +++ b/src/server/lib/jsonschema/schemas/1.0.0.json @@ -44,9 +44,7 @@ "type": "number" } }, - "required": [ - "name" - ] + "required": ["name"] } }, "optionalServices": { @@ -69,9 +67,7 @@ "type": "number" } }, - "required": [ - "name" - ] + "required": ["name"] } }, "webhooks": { @@ -101,12 +97,6 @@ "trigger": { "type": "string" }, - "env": { - "type": "object", - "required": [ - "branch" - ] - }, "docker": { "type": "object", "additionalProperties": false, @@ -130,9 +120,7 @@ "type": "number" } }, - "required": [ - "image" - ] + "required": ["image"] }, "command": { "type": "object", @@ -148,19 +136,13 @@ "type": "number" } }, - "required": [ - "image", - "script" - ] + "required": ["image", "script"] + }, + "env": { + "type": "object" } }, - "required": [ - "state", - "type", - "env", - "trigger", - "env" - ] + "required": ["state", "type", "env"] } } } @@ -192,9 +174,71 @@ "type": "string" } }, - "required": [ - "name" - ] + "required": ["name"] + } + }, + "deploymentDependsOn": { + "type": "array", + "minItems": 0, + "items": { + "type": "string" + } + }, + "kedaScaleToZero": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string" + }, + "replicas": { + "type": "object", + "additionalProperties": false, + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + } + }, + "scaledownPeriod": { + "type": "number" + }, + "maxRetries": { + "type": "number" + }, + "scalingMetric": { + "type": "object", + "additionalProperties": false, + "properties": { + "requestRate": { + "type": "object", + "additionalProperties": false, + "properties": { + "granularity": { + "type": "string" + }, + "targetValue": { + "type": "number" + }, + "window": { + "type": "string" + } + } + }, + "concurrency": { + "type": "object", + "additionalProperties": false, + "properties": { + "targetValue": { + "type": "number" + } + } + } + } + } } }, "helm": { @@ -248,9 +292,10 @@ } } }, - "required": [ - "name" - ] + "required": ["name"] + }, + "envLens": { + "type": "boolean" }, "grpc": { "type": "boolean" @@ -271,6 +316,18 @@ "pipelineId": { "type": "string" }, + "ecr": { + "type": "string" + }, + "builder": { + "type": "object", + "additionalProperties": true, + "properties": { + "engine": { + "type": "string" + } + } + }, "app": { "type": "object", "additionalProperties": false, @@ -307,9 +364,7 @@ "minItems": 1 } }, - "required": [ - "dockerfilePath" - ] + "required": ["dockerfilePath"] }, "init": { "type": "object", @@ -328,36 +383,12 @@ "type": "object" } }, - "required": [ - "dockerfilePath" - ] - }, - "ecr": { - "type": "string" - }, - "builder": { - "type": "object", - "additionalProperties": true, - "properties": { - "engine": { - "type": "string" - } - } + "required": ["dockerfilePath"] } }, - "required": [ - "defaultTag", - "app" - ] - }, - "envLens": { - "type": "boolean" + "required": ["defaultTag", "app"] } - }, - "required": [ - "repository", - "branchName" - ] + } }, "codefresh": { "type": "object", @@ -527,18 +558,6 @@ } } }, - "scaling": { - "type": "object", - "additionalProperties": false, - "properties": { - "scaleToZero": { - "type": "boolean" - }, - "scaleToZeroMetricsCheckInterval": { - "type": "number" - } - } - }, "hostnames": { "type": "object", "additionalProperties": false, @@ -559,6 +578,7 @@ }, "network": { "type": "object", + "additionalProperties": true, "properties": { "ipWhitelist": { "type": "array" @@ -569,9 +589,6 @@ "hostPortMapping": { "type": "object" }, - "kuma": { - "type": "object" - }, "grpc": { "type": "object", "additionalProperties": false, @@ -587,8 +604,7 @@ } } } - }, - "additionalProperties": false + } }, "serviceDisks": { "type": "array", @@ -614,20 +630,13 @@ "type": "string" } }, - "required": [ - "name", - "mountPath", - "storageSize" - ] + "required": ["name", "mountPath", "storageSize"] } } } } }, - "required": [ - "repository", - "branchName" - ] + "required": ["repository", "branchName"] }, "github": { "type": "object", @@ -649,6 +658,18 @@ "pipelineId": { "type": "string" }, + "ecr": { + "type": "string" + }, + "builder": { + "type": "object", + "additionalProperties": true, + "properties": { + "engine": { + "type": "string" + } + } + }, "app": { "type": "object", "additionalProperties": false, @@ -685,9 +706,7 @@ "minItems": 1 } }, - "required": [ - "dockerfilePath" - ] + "required": ["dockerfilePath"] }, "init": { "type": "object", @@ -706,27 +725,10 @@ "type": "object" } }, - "required": [ - "dockerfilePath" - ] - }, - "ecr": { - "type": "string" - }, - "builder": { - "type": "object", - "additionalProperties": true, - "properties": { - "engine": { - "type": "string" - } - } + "required": ["dockerfilePath"] } }, - "required": [ - "defaultTag", - "app" - ] + "required": ["defaultTag", "app"] }, "deployment": { "type": "object", @@ -859,18 +861,6 @@ } } }, - "scaling": { - "type": "object", - "additionalProperties": false, - "properties": { - "scaleToZero": { - "type": "boolean" - }, - "scaleToZeroMetricsCheckInterval": { - "type": "number" - } - } - }, "hostnames": { "type": "object", "additionalProperties": false, @@ -891,6 +881,7 @@ }, "network": { "type": "object", + "additionalProperties": true, "properties": { "ipWhitelist": { "type": "array" @@ -901,9 +892,6 @@ "hostPortMapping": { "type": "object" }, - "kuma": { - "type": "object" - }, "grpc": { "type": "object", "additionalProperties": false, @@ -919,8 +907,7 @@ } } } - }, - "additionalProperties": false + } }, "serviceDisks": { "type": "array", @@ -946,21 +933,13 @@ "type": "string" } }, - "required": [ - "name", - "mountPath", - "storageSize" - ] + "required": ["name", "mountPath", "storageSize"] } } } } }, - "required": [ - "repository", - "branchName", - "docker" - ] + "required": ["repository", "branchName", "docker"] }, "docker": { "type": "object", @@ -1115,18 +1094,6 @@ } } }, - "scaling": { - "type": "object", - "additionalProperties": false, - "properties": { - "scaleToZero": { - "type": "boolean" - }, - "scaleToZeroMetricsCheckInterval": { - "type": "number" - } - } - }, "hostnames": { "type": "object", "additionalProperties": false, @@ -1147,6 +1114,7 @@ }, "network": { "type": "object", + "additionalProperties": true, "properties": { "ipWhitelist": { "type": "array" @@ -1157,9 +1125,6 @@ "hostPortMapping": { "type": "object" }, - "kuma": { - "type": "object" - }, "grpc": { "type": "object", "additionalProperties": false, @@ -1175,8 +1140,7 @@ } } } - }, - "additionalProperties": false + } }, "serviceDisks": { "type": "array", @@ -1202,20 +1166,13 @@ "type": "string" } }, - "required": [ - "name", - "mountPath", - "storageSize" - ] + "required": ["name", "mountPath", "storageSize"] } } } } }, - "required": [ - "dockerImage", - "defaultTag" - ] + "required": ["dockerImage", "defaultTag"] }, "externalHttp": { "type": "object", @@ -1228,10 +1185,7 @@ "type": "string" } }, - "required": [ - "defaultInternalHostname", - "defaultPublicUrl" - ] + "required": ["defaultInternalHostname", "defaultPublicUrl"] }, "auroraRestore": { "type": "object", @@ -1244,10 +1198,7 @@ "type": "string" } }, - "required": [ - "command", - "arguments" - ] + "required": ["command", "arguments"] }, "configuration": { "type": "object", @@ -1260,83 +1211,11 @@ "type": "string" } }, - "required": [ - "defaultTag", - "branchName" - ] - }, - "deploymentDependsOn": { - "type": "array", - "minItems": 0, - "items": { - "type": "string" - } - }, - "kedaScaleToZero": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { - "type": "string" - }, - "replicas": { - "type": "object", - "additionalProperties": false, - "properties": { - "min": { - "type": "number" - }, - "max": { - "type": "number" - } - } - }, - "scaledownPeriod": { - "type": "number" - }, - "maxRetries": { - "type": "number" - }, - "scalingMetric": { - "type": "object", - "additionalProperties": false, - "properties": { - "requestRate": { - "type": "object", - "additionalProperties": false, - "properties": { - "granularity": { - "type": "string" - }, - "targetValue": { - "type": "number" - }, - "window": { - "type": "string" - } - } - }, - "concurrency": { - "type": "object", - "additionalProperties": false, - "properties": { - "targetValue": { - "type": "number" - } - } - } - } - } - } + "required": ["defaultTag", "branchName"] } }, - "required": [ - "name" - ] + "required": ["name"] } } - }, - "required": [ - "version" - ] -} \ No newline at end of file + } +} diff --git a/src/server/lib/kubernetes.ts b/src/server/lib/kubernetes.ts index 935f0c05..c6d39769 100644 --- a/src/server/lib/kubernetes.ts +++ b/src/server/lib/kubernetes.ts @@ -1601,6 +1601,34 @@ export function generateDeployManifest({ const manifests: string[] = []; const enableFullYaml = build.enableFullYaml; + // ExternalName service for CLI deploys + // return the ExternalName service if we have a cname + if (CLIDeployTypes.has(deploy.deployable?.type)) { + const externalHost = deploy.cname; + if (externalHost) { + return yaml.dump({ + apiVersion: 'v1', + kind: 'Service', + metadata: { + namespace, + name: deploy.uuid, + labels: { + name: build.uuid, + lc_uuid: build.uuid, + deploy_uuid: deploy.uuid, + }, + }, + spec: { + type: 'ExternalName', + externalName: externalHost, + }, + }); + } else { + logger.info(`[DEPLOY ${deploy.uuid}] No manifest generated for deploy`); + return ''; + } + } + // Reuse existing PVC generation logic const pvcManifests = generatePersistentDisks([deploy], build.uuid, enableFullYaml, namespace); if (pvcManifests) manifests.push(pvcManifests); @@ -1631,32 +1659,6 @@ export function generateDeployManifest({ const grpcManifests = generateGRPCMappings([deploy], build.uuid, enableFullYaml, namespace); if (grpcManifests) manifests.push(grpcManifests); - // ExternalName service for CLI deploys - if (CLIDeployTypes.has(enableFullYaml ? deploy.deployable?.type : deploy.service?.type)) { - const externalHost = enableFullYaml ? deploy.deployable?.externalHost : deploy.service?.externalHost; - if (externalHost) { - manifests.push( - yaml.dump({ - apiVersion: 'v1', - kind: 'Service', - metadata: { - namespace, - name: deploy.uuid, - labels: { - name: build.uuid, - lc_uuid: build.uuid, - deploy_uuid: deploy.uuid, - }, - }, - spec: { - type: 'ExternalName', - externalName: externalHost, - }, - }) - ); - } - } - return manifests.filter((m) => m).join('---\n'); } diff --git a/src/server/lib/tests/__snapshots__/time.spec.ts.snap b/src/server/lib/tests/__snapshots__/time.spec.ts.snap index ae2e826f..a67f480c 100644 --- a/src/server/lib/tests/__snapshots__/time.spec.ts.snap +++ b/src/server/lib/tests/__snapshots__/time.spec.ts.snap @@ -1,5 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Time utils #getTimestamp returns formatted date 1`] = `"2019-07-04 00:00:00"`; +exports[`Time utils #getTimestamp returns formatted date 1`] = `"2019-07-04 00:00:00.0000"`; exports[`Time utils #getTimestamp takes custom formatting string 1`] = `"07/04/2019"`; diff --git a/src/server/lib/time.ts b/src/server/lib/time.ts index 3b96e337..bd82d3a0 100644 --- a/src/server/lib/time.ts +++ b/src/server/lib/time.ts @@ -16,10 +16,7 @@ import moment, { Moment } from 'moment'; -export function getTimestamp( - time: Moment | string, - format = 'YYYY-MM-DD HH:mm:ss' -) { +export function getTimestamp(time: Moment | string, format = 'YYYY-MM-DD HH:mm:ss.SSSS') { const instance = moment.isMoment(time) ? time : moment(time); return instance.format(format); } @@ -28,9 +25,6 @@ export function getUtcTimestamp(time: string | Moment = moment()) { return getTimestamp(moment.utc(time)); } -export function isTimestampExpired( - date: string | Moment, - now: string | Moment = moment() -) { +export function isTimestampExpired(date: string | Moment, now: string | Moment = moment()) { return moment.utc(now).isAfter(moment.utc(date)); } diff --git a/src/server/lib/yamlSchemas/schema_1_0_0/deployment.ts b/src/server/lib/yamlSchemas/schema_1_0_0/deployment.ts index b6dbc4ed..b3e3d068 100644 --- a/src/server/lib/yamlSchemas/schema_1_0_0/deployment.ts +++ b/src/server/lib/yamlSchemas/schema_1_0_0/deployment.ts @@ -98,7 +98,7 @@ export const deployment = { }, network: { type: 'object', - additionalProperties: false, + additionalProperties: true, properties: { ipWhitelist: { type: 'array' }, pathPortMapping: { type: 'object' }, diff --git a/src/server/services/build.ts b/src/server/services/build.ts index 4c852152..2d653eff 100644 --- a/src/server/services/build.ts +++ b/src/server/services/build.ts @@ -51,8 +51,8 @@ const tracer = Tracer.getInstance(); tracer.initialize('build-service'); export interface IngressConfiguration { host: string; + altHosts?: string[]; serviceHost: string; - acmARN: string; deployUUID: string; ipWhitelist: string[]; pathPortMapping: Record; @@ -189,7 +189,6 @@ export default class BuildService extends BaseService { return Object.keys(deployable.hostPortMapping).map((key) => { return { host: `${key}-${this.db.services.Deploy.hostForDeployableDeploy(deploy, deployable)}`, - acmARN: this.db.services.Deploy.acmARNForDeploy(deploy, build.enableFullYaml), deployUUID: `${key}-${deploy.uuid}`, serviceHost: `${deploy.uuid}`, ipWhitelist: deploy.deployable.ipWhitelist, @@ -203,7 +202,6 @@ export default class BuildService extends BaseService { return [ { host: `${this.db.services.Deploy.hostForDeployableDeploy(deploy, deployable)}`, - acmARN: this.db.services.Deploy.acmARNForDeploy(deploy, build.enableFullYaml), deployUUID: `${deploy.uuid}`, serviceHost: `${deploy.uuid}`, ipWhitelist: deploy.deployable.ipWhitelist, @@ -215,7 +213,6 @@ export default class BuildService extends BaseService { return [ { host: this.db.services.Deploy.hostForDeployableDeploy(deploy, deployable), - acmARN: this.db.services.Deploy.acmARNForDeploy(deploy, build.enableFullYaml), deployUUID: deploy.uuid, serviceHost: `${deploy.uuid}`, ipWhitelist: deploy.deployable.ipWhitelist, @@ -231,7 +228,6 @@ export default class BuildService extends BaseService { return Object.keys(service.hostPortMapping).map((key) => { return { host: `${key}-${this.db.services.Deploy.hostForServiceDeploy(deploy, service)}`, - acmARN: this.db.services.Deploy.acmARNForDeploy(deploy, build.enableFullYaml), deployUUID: `${key}-${deploy.uuid}`, serviceHost: `${deploy.uuid}`, ipWhitelist: service.ipWhitelist, @@ -244,7 +240,6 @@ export default class BuildService extends BaseService { return [ { host: `${this.db.services.Deploy.hostForServiceDeploy(deploy, service)}`, - acmARN: this.db.services.Deploy.acmARNForDeploy(deploy, build.enableFullYaml), deployUUID: `${deploy.uuid}`, serviceHost: `${deploy.uuid}`, ipWhitelist: deploy.service.ipWhitelist, @@ -255,7 +250,6 @@ export default class BuildService extends BaseService { return [ { host: this.db.services.Deploy.hostForServiceDeploy(deploy, service), - acmARN: this.db.services.Deploy.acmARNForDeploy(deploy, build.enableFullYaml), deployUUID: deploy.uuid, serviceHost: `${deploy.uuid}`, ipWhitelist: deploy.service.ipWhitelist, @@ -290,10 +284,6 @@ export default class BuildService extends BaseService { return this.domainsAndCertificatesForBuild(build, allServices); } - async deployManually(environmentId: string) { - logger.debug(environmentId); - } - public async createBuildAndDeploys({ repositoryId, repositoryBranchName, @@ -965,7 +955,9 @@ export default class BuildService extends BaseService { // Use DeploymentManager for all active deploys (both Helm and GitHub types) if (activeDeploys.length > 0) { - const deploymentManager = new DeploymentManager(activeDeploys); + // we should ignore Codefresh services here since we dont deploy anything + const managedDeploys = activeDeploys.filter((d) => d.deployable.type !== DeployTypes.CODEFRESH); + const deploymentManager = new DeploymentManager(managedDeploys); await deploymentManager.deploy(); } diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index 8558179e..eee5b222 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -339,6 +339,7 @@ export default class DeployService extends BaseService { await deploy.$query().patch({ status: DeployStatus.BUILDING, buildLogs: uuid, + runUUID: nanoid(), }); logger.info(`[DEPLOY ${deploy?.uuid}] Restoring Aurora cluster for ${deploy?.uuid}`); await cli.cliDeploy(deploy); diff --git a/src/server/services/ingress.ts b/src/server/services/ingress.ts index c7a67c35..f6ab9935 100644 --- a/src/server/services/ingress.ts +++ b/src/server/services/ingress.ts @@ -19,7 +19,6 @@ import rootLogger from 'server/lib/logger'; import BaseService from './_service'; import fs from 'fs'; import { TMP_PATH, QUEUE_NAMES } from 'shared/config'; -import _ from 'lodash'; import { IngressConfiguration } from '../../server/services/build'; import { shellPromise } from 'server/lib/shell'; import yaml from 'js-yaml'; @@ -93,13 +92,13 @@ export default class IngressService extends BaseService { // We just want to create/update ingress for active services only const configurations = await this.db.services.BuildService.configurationsForBuildId(buildId, false); const namespace = await this.db.services.BuildService.getNamespace({ id: buildId }); - const { lifecycleDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); + const { lifecycleDefaults, domainDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); const manifests = configurations.map((configuration) => { return yaml.dump( this.generateNginxManifestForConfiguration({ configuration, - defaultUUID: lifecycleDefaults?.defaultUUID, ingressClassName: lifecycleDefaults?.ingressClassName, + altHosts: domainDefaults?.altHttp || [], }), { skipInvalid: true, @@ -119,12 +118,12 @@ export default class IngressService extends BaseService { */ private generateNginxManifestForConfiguration = ({ configuration, - defaultUUID, ingressClassName, + altHosts, }: { configuration: IngressConfiguration; - defaultUUID: string; ingressClassName?: string; + altHosts: string[]; }) => { const annotations = { ...configuration.ingressAnnotations, @@ -143,7 +142,7 @@ export default class IngressService extends BaseService { }, }, spec: { - rules: this.generateRulesForManifest(configuration), + rules: this.generateRulesForManifest(configuration, altHosts), ingressClassName: ingressClassName || 'nginx', }, }; @@ -153,31 +152,31 @@ export default class IngressService extends BaseService { * Generates the rules for an ingress configuration * @param configuration the ingress configuration to generate rules for */ - private generateRulesForManifest = (configuration: IngressConfiguration) => { - return _.flatten( - Object.entries(configuration.pathPortMapping).map((entry) => { - return [ + private generateRulesForManifest = (configuration: IngressConfiguration, altHosts: string[]) => { + const allHosts = [configuration.host, ...altHosts.map((v) => `${configuration.deployUUID}.${v}`)]; + + const createRule = (host: string, path: string, port: number) => ({ + host, + http: { + paths: [ { - host: `${configuration.host}`, - http: { - paths: [ - { - path: entry[0], - pathType: 'ImplementationSpecific', - backend: { - service: { - name: configuration.serviceHost, - port: { - number: entry[1], - }, - }, - }, + path, + pathType: 'ImplementationSpecific', + backend: { + service: { + name: configuration.serviceHost, + port: { + number: port, }, - ], + }, }, }, - ]; - }) + ], + }, + }); + + return allHosts.flatMap((host) => + Object.entries(configuration.pathPortMapping).map(([path, port]) => createRule(host, path, port)) ); }; diff --git a/src/server/services/types/globalConfig.ts b/src/server/services/types/globalConfig.ts index 0f5dd522..a86e7705 100644 --- a/src/server/services/types/globalConfig.ts +++ b/src/server/services/types/globalConfig.ts @@ -72,6 +72,8 @@ export type DatabaseSettings = { export type DomainDefaults = { http: string; grpc: string; + altHttp?: string[]; + altGrpc?: string[]; }; export type LifecycleIgnores = { diff --git a/src/shared/constants.ts b/src/shared/constants.ts index b1dc2855..edf0d195 100644 --- a/src/shared/constants.ts +++ b/src/shared/constants.ts @@ -20,7 +20,6 @@ export enum DeployTypes { GITHUB = 'github', EXTERNAL_HTTP = 'externalHTTP', AURORA_RESTORE = 'aurora-restore', - RDS_RESTORE = 'rds-restore', CODEFRESH = 'codefresh', CONFIGURATION = 'configuration', HELM = 'helm',