From f39691551636b438ebb6961946819cc4af2d0565 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Sun, 30 Jul 2023 22:25:00 +0530 Subject: [PATCH 01/11] add --no-cache --virtual --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 79656d1f..d31ab08d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /go/bin/cirunner FROM docker:20.10.24-dind # All these steps will be cached #RUN apk add --no-cache ca-certificates -RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && apk add openssh +RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && apk add --no-cache --virtual openssh RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN apk -Uuv add groff less python3 py3-pip RUN pip3 install awscli From 7ade84ed7e385d6fee57fa57776d42d19b957f41 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 3 Aug 2023 10:29:59 +0530 Subject: [PATCH 02/11] run scanning and post ci script for webhook --- App.go | 2 ++ ciStages.go | 36 ++++++++++++++++++++++++++++++++++++ helper/EventHelper.go | 1 + util/CommonConstants.go | 1 + 4 files changed, 40 insertions(+) diff --git a/App.go b/App.go index 459aaee1..667ec23c 100644 --- a/App.go +++ b/App.go @@ -93,6 +93,8 @@ func processEvent(args string) { defer handleCleanup(*ciCdRequest, &exitCode, util.Source_Defer) if ciCdRequest.Type == util.CIEVENT { HandleCIEvent(ciCdRequest, &exitCode) + } else if ciCdRequest.Type == util.WEBHOOK { + runScanningAndPostCiSteps(ciCdRequest) } else { HandleCDEvent(ciCdRequest, &exitCode) } diff --git a/ciStages.go b/ciStages.go index cfab3827..44e8698c 100644 --- a/ciStages.go +++ b/ciStages.go @@ -269,6 +269,42 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e return artifactUploaded, nil } +func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { + log.Println("runScanningAndPostCiSteps", ciCdRequest) + refStageMap := make(map[int][]*helper.StepObject) + scriptEnvs, err := getGlobalEnvVariables(ciCdRequest) + if err != nil { + return err + } + for _, ref := range ciCdRequest.CiRequest.RefPlugins { + refStageMap[ref.Id] = ref.Steps + } + digest, err := helper.ExtractDigestUsingPull(ciCdRequest.CiRequest.Image) + if len(ciCdRequest.CiRequest.PostCiSteps) > 0 { + util.LogStage("running PRE-CI steps") + // run pre artifact processing + _, _, err := RunCiCdSteps(STEP_TYPE_PRE, ciCdRequest.CiRequest.PostCiSteps, refStageMap, scriptEnvs, nil) + if err != nil { + log.Println(err) + return err + + } + } + if ciCdRequest.CiRequest.ScanEnabled { + util.LogStage("IMAGE SCAN") + log.Println(util.DEVTRON, " /image-scanner") + scanEvent := &helper.ScanEvent{Image: ciCdRequest.CiRequest.Image, ImageDigest: digest} + err = helper.SendEventToClairUtility(scanEvent) + if err != nil { + log.Println(err) + return err + + } + log.Println(util.DEVTRON, " /image-scanner") + } + return nil +} + func getPostCiStepToRunOnCiFail(postCiSteps []*helper.StepObject) []*helper.StepObject { var postCiStepsToTriggerOnCiFail []*helper.StepObject if len(postCiSteps) > 0 { diff --git a/helper/EventHelper.go b/helper/EventHelper.go index aafa6d81..9f92971a 100644 --- a/helper/EventHelper.go +++ b/helper/EventHelper.go @@ -141,6 +141,7 @@ type CiRequest struct { OrchestratorToken string `json:"orchestratorToken"` ImageRetryCount int `json:"imageRetryCount"` ImageRetryInterval int `json:"imageRetryInterval"` + Image string `json:"image"` } type CdRequest struct { diff --git a/util/CommonConstants.go b/util/CommonConstants.go index 39a96450..103ae126 100644 --- a/util/CommonConstants.go +++ b/util/CommonConstants.go @@ -30,6 +30,7 @@ const LOCAL_BUILDX_CACHE_LOCATION = LOCAL_BUILDX_LOCATION + "/cache" const CIEVENT = "CI" const CDSTAGE = "CD" +const WEBHOOK = "WEBHOOK" const DRY_RUN = "DryRun" const ENV_VARIABLE_BUILD_SUCCESS = "BUILD_SUCCESS" From 75949ec6af19c9424c36f255b528011ef7495197 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 3 Aug 2023 17:26:00 +0530 Subject: [PATCH 03/11] print Devtron --- ciStages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ciStages.go b/ciStages.go index 44e8698c..bb8f8ad5 100644 --- a/ciStages.go +++ b/ciStages.go @@ -270,7 +270,7 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e } func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { - log.Println("runScanningAndPostCiSteps", ciCdRequest) + log.Println(util.DEVTRON, "runScanningAndPostCiSteps", ciCdRequest) refStageMap := make(map[int][]*helper.StepObject) scriptEnvs, err := getGlobalEnvVariables(ciCdRequest) if err != nil { From cca1411f3dcfb7d116591f0bd37a46f6a666be74 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 3 Aug 2023 19:05:01 +0530 Subject: [PATCH 04/11] added logs --- ciStages.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ciStages.go b/ciStages.go index bb8f8ad5..1d7c0a69 100644 --- a/ciStages.go +++ b/ciStages.go @@ -272,18 +272,20 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { log.Println(util.DEVTRON, "runScanningAndPostCiSteps", ciCdRequest) refStageMap := make(map[int][]*helper.StepObject) - scriptEnvs, err := getGlobalEnvVariables(ciCdRequest) - if err != nil { - return err - } for _, ref := range ciCdRequest.CiRequest.RefPlugins { refStageMap[ref.Id] = ref.Steps } + log.Println(util.DEVTRON, "ExtractDigestUsingPull", ciCdRequest.CiRequest.Image) digest, err := helper.ExtractDigestUsingPull(ciCdRequest.CiRequest.Image) + if err != nil { + log.Println(util.DEVTRON, "Error in digest", err) + return err + } + log.Println(util.DEVTRON, "ExtractDigestUsingPull -> ", digest) if len(ciCdRequest.CiRequest.PostCiSteps) > 0 { util.LogStage("running PRE-CI steps") // run pre artifact processing - _, _, err := RunCiCdSteps(STEP_TYPE_PRE, ciCdRequest.CiRequest.PostCiSteps, refStageMap, scriptEnvs, nil) + _, _, err := RunCiCdSteps(STEP_TYPE_PRE, ciCdRequest.CiRequest.PostCiSteps, refStageMap, nil, nil) if err != nil { log.Println(err) return err From ad8201a5bc444288419886725d1a468b189f72d4 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 3 Aug 2023 19:11:56 +0530 Subject: [PATCH 05/11] change pre to post --- App.go | 4 +++- ciStages.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/App.go b/App.go index 667ec23c..fb81c6e8 100644 --- a/App.go +++ b/App.go @@ -44,6 +44,8 @@ func handleCleanup(ciCdRequest helper.CiCdTriggerEvent, exitCode *int, source st } func main() { + args := `{"type":"WEBHOOK","ciRequest":{"workflowNamePrefix":"ci","pipelineName":"","pipelineId":4,"dockerImageTag":"","dockerRegistryId":"","dockerRegistryType":"","dockerRegistryURL":"","dockerConnection":"","dockerCert":"","dockerRepository":"","checkoutPath":"","dockerUsername":"","dockerPassword":"","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"","ciProjectDetails":null,"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":0,"ciImage":"ashexp/test:002a2133-47-115","namespace":"devtron-ci","workflowId":0,"triggeredBy":4,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","ciArtifactBucket":"","ciArtifactFileName":"","ciArtifactRegion":"","scanEnabled":true,"cloudProvider":"S3","blobStorageConfigured":false,"blobStorageS3Config":null,"azureBlobConfig":null,"gcpBlobConfig":null,"blobStorageLogsKey":"arsenal-v1/ci","inAppLoggingEnabled":false,"defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":null,"postCiSteps":[{"name":"Task1","index":1,"stepType":"INLINE","executorType":"SHELL","script":"#!/bin/sh\nset -eo pipefail \n#set -v ## uncomment this to debug the script\n\necho \"HelloWorld\"","inputVars":null,"exposedPorts":null,"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMount":null,"sourceCodeMount":null,"extraVolumeMounts":null,"artifactPaths":null,"triggerIfParentStageFail":false}],"refPlugins":null,"appName":"","triggerByAuthor":"","ciBuildConfig":null,"ciBuildDockerMtuValue":-1,"ignoreDockerCachePush":false,"ignoreDockerCachePull":false,"cacheInvalidate":false,"IsPvcMounted":false,"extraEnvironmentVariables":null,"enableBuildContext":false,"appId":55,"environmentId":0,"orchestratorHost":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","orchestratorToken":"","isExtRun":false,"imageRetryCount":0,"imageRetryInterval":5,"image":"ashexp/test:002a2133-47-115"},"cdRequest":null}` + //args := `{"type":"CI","ciRequest":{"DockerBuildTargetPlatform":"linux/arm64", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/getting-started-nodejs","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"da3ba3254712965b5944a6271e71bff91fe51f20","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` //args := `{"type":"DryRun","dryRunRequest":{"buildPackParams":{"builderId":"gcr.io/buildpacks/builder:v1"},"DockerBuildTargetPlatform":"", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/sample-go-app","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"8654623ec2bd9efd663935cb8332c8c765541837","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` //' {"workflowNamePrefix":"55-suraj-23-ci-suraj-test-pipeline-8","pipelineName":"suraj-23-ci-suraj-test-pipeline","pipelineId":8,"dockerImageTag":"a6b809c4be87c217feba4af15cf5ebc3cafe21e0","dockerRegistryURL":"686244538589.dkr.ecr.us-east-2.amazonaws.com","dockerRepository":"test/suraj-23","dockerfileLocation":"./notifier/Dockerfile","awsRegion":"us-east-2","ciCacheLocation":"ci-caching","ciCacheFileName":"suraj-23-ci-suraj-test-pipeline.tar.gz","ciProjectDetails":[{"gitRepository":"https://gitlab.com/devtron/notifier.git","materialName":"1-notifier","checkoutPath":"./notifier","commitHash":"d4df38bcd065004014d255c2203d592a91585955","commitTime":"0001-01-01T00:00:00Z","branch":"ci_with_argo","type":"SOURCE_TYPE_BRANCH_FIXED","message":"test-commit","gitOptions":{"userName":"Suraj24","password":"Devtron@1234","sshKey":"","accessToken":"","authMode":"USERNAME_PASSWORD"}},{"gitRepository":"https://gitlab.com/devtron/orchestrator.git","materialName":"2-orchestrator","checkoutPath":"./orch","commitHash":"","commitTime":"0001-01-01T00:00:00Z","branch":"ci_with_argo","type":"SOURCE_TYPE_BRANCH_FIXED","message":"","gitOptions":{"userName":"Suraj24","password":"Devtron@1234","sshKey":"","accessToken":"","authMode":""}}],"ciImage":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:latest","namespace":"default"}' @@ -58,7 +60,7 @@ func main() { util.SpawnProcessWithLogging() } - args := os.Getenv(util.CiCdEventEnvKey) + //args := os.Getenv(util.CiCdEventEnvKey) processEvent(args) } diff --git a/ciStages.go b/ciStages.go index 1d7c0a69..8f1199ea 100644 --- a/ciStages.go +++ b/ciStages.go @@ -285,7 +285,7 @@ func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { if len(ciCdRequest.CiRequest.PostCiSteps) > 0 { util.LogStage("running PRE-CI steps") // run pre artifact processing - _, _, err := RunCiCdSteps(STEP_TYPE_PRE, ciCdRequest.CiRequest.PostCiSteps, refStageMap, nil, nil) + _, _, err := RunCiCdSteps(STEP_TYPE_POST, ciCdRequest.CiRequest.PostCiSteps, refStageMap, nil, nil) if err != nil { log.Println(err) return err From 05d1f0513f5445efda6094a7b9bac4968dad88da Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 3 Aug 2023 19:27:11 +0530 Subject: [PATCH 06/11] args change --- App.go | 4 ++-- ciStages.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/App.go b/App.go index fb81c6e8..cf260bdd 100644 --- a/App.go +++ b/App.go @@ -44,7 +44,7 @@ func handleCleanup(ciCdRequest helper.CiCdTriggerEvent, exitCode *int, source st } func main() { - args := `{"type":"WEBHOOK","ciRequest":{"workflowNamePrefix":"ci","pipelineName":"","pipelineId":4,"dockerImageTag":"","dockerRegistryId":"","dockerRegistryType":"","dockerRegistryURL":"","dockerConnection":"","dockerCert":"","dockerRepository":"","checkoutPath":"","dockerUsername":"","dockerPassword":"","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"","ciProjectDetails":null,"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":0,"ciImage":"ashexp/test:002a2133-47-115","namespace":"devtron-ci","workflowId":0,"triggeredBy":4,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","ciArtifactBucket":"","ciArtifactFileName":"","ciArtifactRegion":"","scanEnabled":true,"cloudProvider":"S3","blobStorageConfigured":false,"blobStorageS3Config":null,"azureBlobConfig":null,"gcpBlobConfig":null,"blobStorageLogsKey":"arsenal-v1/ci","inAppLoggingEnabled":false,"defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":null,"postCiSteps":[{"name":"Task1","index":1,"stepType":"INLINE","executorType":"SHELL","script":"#!/bin/sh\nset -eo pipefail \n#set -v ## uncomment this to debug the script\n\necho \"HelloWorld\"","inputVars":null,"exposedPorts":null,"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMount":null,"sourceCodeMount":null,"extraVolumeMounts":null,"artifactPaths":null,"triggerIfParentStageFail":false}],"refPlugins":null,"appName":"","triggerByAuthor":"","ciBuildConfig":null,"ciBuildDockerMtuValue":-1,"ignoreDockerCachePush":false,"ignoreDockerCachePull":false,"cacheInvalidate":false,"IsPvcMounted":false,"extraEnvironmentVariables":null,"enableBuildContext":false,"appId":55,"environmentId":0,"orchestratorHost":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","orchestratorToken":"","isExtRun":false,"imageRetryCount":0,"imageRetryInterval":5,"image":"ashexp/test:002a2133-47-115"},"cdRequest":null}` + //args := `{"type":"WEBHOOK","ciRequest":{"workflowNamePrefix":"ci","pipelineName":"","pipelineId":4,"dockerImageTag":"","dockerRegistryId":"","dockerRegistryType":"","dockerRegistryURL":"","dockerConnection":"","dockerCert":"","dockerRepository":"","checkoutPath":"","dockerUsername":"","dockerPassword":"","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"","ciProjectDetails":null,"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":0,"ciImage":"ashexp/test:002a2133-47-115","namespace":"devtron-ci","workflowId":0,"triggeredBy":4,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","ciArtifactBucket":"","ciArtifactFileName":"","ciArtifactRegion":"","scanEnabled":true,"cloudProvider":"S3","blobStorageConfigured":false,"blobStorageS3Config":null,"azureBlobConfig":null,"gcpBlobConfig":null,"blobStorageLogsKey":"arsenal-v1/ci","inAppLoggingEnabled":false,"defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":null,"postCiSteps":[{"name":"Task1","index":1,"stepType":"INLINE","executorType":"SHELL","script":"#!/bin/sh\nset -eo pipefail \n#set -v ## uncomment this to debug the script\n\necho \"HelloWorld\"","inputVars":null,"exposedPorts":null,"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMount":null,"sourceCodeMount":null,"extraVolumeMounts":null,"artifactPaths":null,"triggerIfParentStageFail":false}],"refPlugins":null,"appName":"","triggerByAuthor":"","ciBuildConfig":null,"ciBuildDockerMtuValue":-1,"ignoreDockerCachePush":false,"ignoreDockerCachePull":false,"cacheInvalidate":false,"IsPvcMounted":false,"extraEnvironmentVariables":null,"enableBuildContext":false,"appId":55,"environmentId":0,"orchestratorHost":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","orchestratorToken":"","isExtRun":false,"imageRetryCount":0,"imageRetryInterval":5,"image":"ashexp/test:002a2133-47-115"},"cdRequest":null}` //args := `{"type":"CI","ciRequest":{"DockerBuildTargetPlatform":"linux/arm64", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/getting-started-nodejs","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"da3ba3254712965b5944a6271e71bff91fe51f20","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` //args := `{"type":"DryRun","dryRunRequest":{"buildPackParams":{"builderId":"gcr.io/buildpacks/builder:v1"},"DockerBuildTargetPlatform":"", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/sample-go-app","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"8654623ec2bd9efd663935cb8332c8c765541837","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` @@ -60,7 +60,7 @@ func main() { util.SpawnProcessWithLogging() } - //args := os.Getenv(util.CiCdEventEnvKey) + args := os.Getenv(util.CiCdEventEnvKey) processEvent(args) } diff --git a/ciStages.go b/ciStages.go index 8f1199ea..6f9fd40e 100644 --- a/ciStages.go +++ b/ciStages.go @@ -283,7 +283,7 @@ func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { } log.Println(util.DEVTRON, "ExtractDigestUsingPull -> ", digest) if len(ciCdRequest.CiRequest.PostCiSteps) > 0 { - util.LogStage("running PRE-CI steps") + util.LogStage("running POST-CI steps") // run pre artifact processing _, _, err := RunCiCdSteps(STEP_TYPE_POST, ciCdRequest.CiRequest.PostCiSteps, refStageMap, nil, nil) if err != nil { From 06cb5829c29fa50ef763fa20832a97e5f20ecbbb Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 4 Aug 2023 10:37:36 +0530 Subject: [PATCH 07/11] remove image scanning --- App.go | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++- ciStages.go | 36 +++++++++--------- 2 files changed, 124 insertions(+), 20 deletions(-) diff --git a/App.go b/App.go index cf260bdd..34e23a31 100644 --- a/App.go +++ b/App.go @@ -44,12 +44,116 @@ func handleCleanup(ciCdRequest helper.CiCdTriggerEvent, exitCode *int, source st } func main() { - //args := `{"type":"WEBHOOK","ciRequest":{"workflowNamePrefix":"ci","pipelineName":"","pipelineId":4,"dockerImageTag":"","dockerRegistryId":"","dockerRegistryType":"","dockerRegistryURL":"","dockerConnection":"","dockerCert":"","dockerRepository":"","checkoutPath":"","dockerUsername":"","dockerPassword":"","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"","ciProjectDetails":null,"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":0,"ciImage":"ashexp/test:002a2133-47-115","namespace":"devtron-ci","workflowId":0,"triggeredBy":4,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","ciArtifactBucket":"","ciArtifactFileName":"","ciArtifactRegion":"","scanEnabled":true,"cloudProvider":"S3","blobStorageConfigured":false,"blobStorageS3Config":null,"azureBlobConfig":null,"gcpBlobConfig":null,"blobStorageLogsKey":"arsenal-v1/ci","inAppLoggingEnabled":false,"defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":null,"postCiSteps":[{"name":"Task1","index":1,"stepType":"INLINE","executorType":"SHELL","script":"#!/bin/sh\nset -eo pipefail \n#set -v ## uncomment this to debug the script\n\necho \"HelloWorld\"","inputVars":null,"exposedPorts":null,"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMount":null,"sourceCodeMount":null,"extraVolumeMounts":null,"artifactPaths":null,"triggerIfParentStageFail":false}],"refPlugins":null,"appName":"","triggerByAuthor":"","ciBuildConfig":null,"ciBuildDockerMtuValue":-1,"ignoreDockerCachePush":false,"ignoreDockerCachePull":false,"cacheInvalidate":false,"IsPvcMounted":false,"extraEnvironmentVariables":null,"enableBuildContext":false,"appId":55,"environmentId":0,"orchestratorHost":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","orchestratorToken":"","isExtRun":false,"imageRetryCount":0,"imageRetryInterval":5,"image":"ashexp/test:002a2133-47-115"},"cdRequest":null}` - + //args := `{"type":"WEBHOOK","ciRequest":{"workflowNamePrefix":"ci","pipelineName":"","pipelineId":4,"dockerImageTag":"","dockerRegistryId":"","dockerRegistryType":"","dockerRegistryURL":"","dockerConnection":"","dockerCert":"","dockerRepository":"","checkoutPath":"","dockerUsername":"","dockerPassword":"","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"","ciProjectDetails":null,"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":0,"ciImage":"ashexp/test:002a2133-47-115","namespace":"devtron-ci","workflowId":0,"triggeredBy":4,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","ciArtifactBucket":"","ciArtifactFileName":"","ciArtifactRegion":"","scanEnabled":true,"cloudProvider":"S3","blobStorageConfigured":false,"blobStorageS3Config":null,"azureBlobConfig":null,"gcpBlobConfig":null,"blobStorageLogsKey":"arsenal-v1/ci","inAppLoggingEnabled":false,"defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":null,"postCiSteps":[{"name":"Task1","index":1,"stepType":"INLINE","executorType":"SHELL","script":"#!/bin/shset -eo pipefail #set -v ## uncomment this to debug the scriptecho \"HelloWorld\"","inputVars":null,"exposedPorts":null,"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMount":null,"sourceCodeMount":null,"extraVolumeMounts":null,"artifactPaths":null,"triggerIfParentStageFail":false}],"refPlugins":null,"appName":"","triggerByAuthor":"","ciBuildConfig":null,"ciBuildDockerMtuValue":-1,"ignoreDockerCachePush":false,"ignoreDockerCachePull":false,"cacheInvalidate":false,"IsPvcMounted":false,"extraEnvironmentVariables":null,"enableBuildContext":false,"appId":55,"environmentId":0,"orchestratorHost":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","orchestratorToken":"","isExtRun":false,"imageRetryCount":0,"imageRetryInterval":5,"image":"ashexp/test:002a2133-47-115"},"cdRequest":null}` + // args := `'{"name":"ci","inputs":{},"outputs":{},"metadata":{},"container":{"name":"","image":"quay.io/devtron/test:75949ec6-890-6954","ports":[{"name":"app-data","containerPort":9102}],"env":[{"name":"IMAGE_SCANNER_ENDPOINT","value":"http://image-scanner-service.devtroncd:80"},{"name":"CI_CD_EVENT","value":"{\"type\":\"WEBHOOK\",\"ciRequest\":{\"workflowNamePrefix\":\"ci\",\"pipelineName\":\"\",\"pipelineId\":4,\"dockerImageTag\":\"\",\"dockerRegistryId\":\"\",\"dockerRegistryType\":\"\",\"dockerRegistryURL\":\"\",\"dockerConnection\":\"\",\"dockerCert\":\"\",\"dockerRepository\":\"\",\"checkoutPath\":\"\",\"dockerUsername\":\"\",\"dockerPassword\":\"\",\"awsRegion\":\"\",\"accessKey\":\"\",\"secretKey\":\"\",\"ciCacheLocation\":\"\",\"ciCacheRegion\":\"\",\"ciCacheFileName\":\"\",\"ciProjectDetails\":null,\"containerResources\":{\"minCpu\":\"\",\"maxCpu\":\"\",\"minStorage\":\"\",\"maxStorage\":\"\",\"minEphStorage\":\"\",\"maxEphStorage\":\"\",\"minMem\":\"\",\"maxMem\":\"\"},\"activeDeadlineSeconds\":0,\"ciImage\":\"quay.io/devtron/test:75949ec6-890-6954\",\"namespace\":\"devtron-ci\",\"workflowId\":0,\"triggeredBy\":4,\"cacheLimit\":5000000000,\"beforeDockerBuildScripts\":null,\"afterDockerBuildScripts\":null,\"ciArtifactLocation\":\"\",\"ciArtifactBucket\":\"\",\"ciArtifactFileName\":\"\",\"ciArtifactRegion\":\"\",\"scanEnabled\":true,\"cloudProvider\":\"\",\"blobStorageConfigured\":false,\"blobStorageS3Config\":null,\"azureBlobConfig\":null,\"gcpBlobConfig\":null,\"blobStorageLogsKey\":\"devtron/ci\",\"inAppLoggingEnabled\":true,\"defaultAddressPoolBaseCidr\":\"\",\"defaultAddressPoolSize\":0,\"preCiSteps\":null,\"postCiSteps\":[{\"name\":\"Task + // 1\",\"index\":1,\"stepType\":\"INLINE\",\"executorType\":\"SHELL\",\"script\":\"#!/bin/sh + // \set -eo pipefail \#set -v ## uncomment this to debug the script + // \\echo \\\"Hello + // World\\\"\",\"inputVars\":null,\"exposedPorts\":null,\"outputVars\":null,\"triggerSkipConditions\":null,\"successFailureConditions\":null,\"dockerImage\":\"\",\"command\":\"\",\"args\":null,\"customScriptMount\":null,\"sourceCodeMount\":null,\"extraVolumeMounts\":null,\"artifactPaths\":null,\"triggerIfParentStageFail\":false}],\"refPlugins\":null,\"appName\":\"\",\"triggerByAuthor\":\"\",\"ciBuildConfig\":null,\"ciBuildDockerMtuValue\":-1,\"ignoreDockerCachePush\":false,\"ignoreDockerCachePull\":false,\"cacheInvalidate\":false,\"IsPvcMounted\":false,\"extraEnvironmentVariables\":null,\"enableBuildContext\":true,\"appId\":55,\"environmentId\":0,\"orchestratorHost\":\"http://20.231.45.51:32267/orchestrator/webhook/msg/nats\",\"orchestratorToken\":\"cCtrbHF2ditXTDZ5UEkza1dVTXUyZXcrUWljPQo\",\"isExtRun\":false,\"imageRetryCount\":0,\"imageRetryInterval\":5,\"image\":\"quay.io/devtron/test:75949ec6-890-6954\"},\"cdRequest\":null}"},{"name":"IN_APP_LOGGING","value":"true"}],"resources":{"limits":{"cpu":"500m","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}},"securityContext":{"privileged":true}},"archiveLocation":{"archiveLogs":false},"activeDeadlineSeconds":2147483632}'` //args := `{"type":"CI","ciRequest":{"DockerBuildTargetPlatform":"linux/arm64", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/getting-started-nodejs","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"da3ba3254712965b5944a6271e71bff91fe51f20","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` //args := `{"type":"DryRun","dryRunRequest":{"buildPackParams":{"builderId":"gcr.io/buildpacks/builder:v1"},"DockerBuildTargetPlatform":"", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/sample-go-app","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"8654623ec2bd9efd663935cb8332c8c765541837","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` //' {"workflowNamePrefix":"55-suraj-23-ci-suraj-test-pipeline-8","pipelineName":"suraj-23-ci-suraj-test-pipeline","pipelineId":8,"dockerImageTag":"a6b809c4be87c217feba4af15cf5ebc3cafe21e0","dockerRegistryURL":"686244538589.dkr.ecr.us-east-2.amazonaws.com","dockerRepository":"test/suraj-23","dockerfileLocation":"./notifier/Dockerfile","awsRegion":"us-east-2","ciCacheLocation":"ci-caching","ciCacheFileName":"suraj-23-ci-suraj-test-pipeline.tar.gz","ciProjectDetails":[{"gitRepository":"https://gitlab.com/devtron/notifier.git","materialName":"1-notifier","checkoutPath":"./notifier","commitHash":"d4df38bcd065004014d255c2203d592a91585955","commitTime":"0001-01-01T00:00:00Z","branch":"ci_with_argo","type":"SOURCE_TYPE_BRANCH_FIXED","message":"test-commit","gitOptions":{"userName":"Suraj24","password":"Devtron@1234","sshKey":"","accessToken":"","authMode":"USERNAME_PASSWORD"}},{"gitRepository":"https://gitlab.com/devtron/orchestrator.git","materialName":"2-orchestrator","checkoutPath":"./orch","commitHash":"","commitTime":"0001-01-01T00:00:00Z","branch":"ci_with_argo","type":"SOURCE_TYPE_BRANCH_FIXED","message":"","gitOptions":{"userName":"Suraj24","password":"Devtron@1234","sshKey":"","accessToken":"","authMode":""}}],"ciImage":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:latest","namespace":"default"}' + // args := `{ + // "type": "WEBHOOK", + // "ciRequest": { + // "workflowNamePrefix": "ci", + // "pipelineName": "", + // "pipelineId": 4, + // "dockerImageTag": "", + // "dockerRegistryId": "", + // "dockerRegistryType": "", + // "dockerRegistryURL": "", + // "dockerConnection": "", + // "dockerCert": "", + // "dockerRepository": "", + // "checkoutPath": "", + // "dockerUsername": "", + // "dockerPassword": "", + // "awsRegion": "", + // "accessKey": "", + // "secretKey": "", + // "ciCacheLocation": "", + // "ciCacheRegion": "", + // "ciCacheFileName": "", + // "ciProjectDetails": null, + // "containerResources": { + // "minCpu": "", + // "maxCpu": "", + // "minStorage": "", + // "maxStorage": "", + // "minEphStorage": "", + // "maxEphStorage": "", + // "minMem": "", + // "maxMem": "" + // }, + // "activeDeadlineSeconds": 0, + // "ciImage": "ashexp/test:002a2133-47-115", + // "namespace": "devtron-ci", + // "workflowId": 0, + // "triggeredBy": 4, + // "cacheLimit": 5000000000, + // "beforeDockerBuildScripts": null, + // "afterDockerBuildScripts": null, + // "ciArtifactLocation": "", + // "ciArtifactBucket": "", + // "ciArtifactFileName": "", + // "ciArtifactRegion": "", + // "scanEnabled": true, + // "cloudProvider": "", + // "blobStorageConfigured": false, + // "blobStorageS3Config": null, + // "azureBlobConfig": null, + // "gcpBlobConfig": null, + // "blobStorageLogsKey": "devtron/ci", + // "inAppLoggingEnabled": true, + // "defaultAddressPoolBaseCidr": "", + // "defaultAddressPoolSize": 0, + // "preCiSteps": null, + // "postCiSteps": [ + // { + // "name": "Task1", + // "index": 1, + // "stepType": "INLINE", + // "executorType": "SHELL", + // "script": "#!/bin/shset -eo pipefail #set -v ## uncomment this to debug the script echo ", + // "inputVars": null, + // "exposedPorts": null, + // "outputVars": null, + // "triggerSkipConditions": null, + // "successFailureConditions": null, + // "dockerImage": "", + // "command": "", + // "args": null, + // "customScriptMount": null, + // "sourceCodeMount": null, + // "extraVolumeMounts": null, + // "artifactPaths": null, + // "triggerIfParentStageFail": false + // } + // ], + // "refPlugins": null, + // "appName": "", + // "triggerByAuthor": "", + // "ciBuildConfig": null, + // "ciBuildDockerMtuValue": -1, + // "ignoreDockerCachePush": false, + // "ignoreDockerCachePull": false, + // "cacheInvalidate": false, + // "IsPvcMounted": false, + // "extraEnvironmentVariables": null, + // "enableBuildContext": true, + // "appId": 55, + // "environmentId": 0, + // "orchestratorHost": "http://20.231.45.51:32267/orchestrator/webhook/msg/nats", + // "orchestratorToken": "cCtrbHF2ditXTDZ5UEkza1dVTXUyZXcrUWljPQo", + // "isExtRun": false, + // "imageRetryCount": 0, + // "imageRetryInterval": 5, + // "image": "ashexp/test:002a2133-47-115" + // }, + // "cdRequest": null + //}` LoggingMode := "RunMode" if len(os.Args) > 1 { LoggingMode = os.Args[1] diff --git a/ciStages.go b/ciStages.go index 6f9fd40e..490664ca 100644 --- a/ciStages.go +++ b/ciStages.go @@ -276,12 +276,12 @@ func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { refStageMap[ref.Id] = ref.Steps } log.Println(util.DEVTRON, "ExtractDigestUsingPull", ciCdRequest.CiRequest.Image) - digest, err := helper.ExtractDigestUsingPull(ciCdRequest.CiRequest.Image) - if err != nil { - log.Println(util.DEVTRON, "Error in digest", err) - return err - } - log.Println(util.DEVTRON, "ExtractDigestUsingPull -> ", digest) + //digest, err := helper.ExtractDigestUsingPull(ciCdRequest.CiRequest.Image) + //if err != nil { + // log.Println(util.DEVTRON, "Error in digest", err) + // return err + //} + //log.Println(util.DEVTRON, "ExtractDigestUsingPull -> ", digest) if len(ciCdRequest.CiRequest.PostCiSteps) > 0 { util.LogStage("running POST-CI steps") // run pre artifact processing @@ -292,18 +292,18 @@ func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { } } - if ciCdRequest.CiRequest.ScanEnabled { - util.LogStage("IMAGE SCAN") - log.Println(util.DEVTRON, " /image-scanner") - scanEvent := &helper.ScanEvent{Image: ciCdRequest.CiRequest.Image, ImageDigest: digest} - err = helper.SendEventToClairUtility(scanEvent) - if err != nil { - log.Println(err) - return err - - } - log.Println(util.DEVTRON, " /image-scanner") - } + //if ciCdRequest.CiRequest.ScanEnabled { + // util.LogStage("IMAGE SCAN") + // log.Println(util.DEVTRON, " /image-scanner") + // scanEvent := &helper.ScanEvent{Image: ciCdRequest.CiRequest.Image, ImageDigest: digest} + // err = helper.SendEventToClairUtility(scanEvent) + // if err != nil { + // log.Println(err) + // return err + // + // } + // log.Println(util.DEVTRON, " /image-scanner") + //} return nil } From a4ff957b27017b83e0ee31165eb84d9933f4e737 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 4 Aug 2023 12:09:35 +0530 Subject: [PATCH 08/11] run scanning and start docker demon --- ciStages.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/ciStages.go b/ciStages.go index 490664ca..697cfb79 100644 --- a/ciStages.go +++ b/ciStages.go @@ -275,13 +275,14 @@ func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { for _, ref := range ciCdRequest.CiRequest.RefPlugins { refStageMap[ref.Id] = ref.Steps } + helper.StartDockerDaemon(ciCdRequest.CiRequest.DockerConnection, ciCdRequest.CiRequest.DockerRegistryURL, ciCdRequest.CiRequest.DockerCert, ciCdRequest.CiRequest.DefaultAddressPoolBaseCidr, ciCdRequest.CiRequest.DefaultAddressPoolSize, ciCdRequest.CiRequest.CiBuildDockerMtuValue) log.Println(util.DEVTRON, "ExtractDigestUsingPull", ciCdRequest.CiRequest.Image) - //digest, err := helper.ExtractDigestUsingPull(ciCdRequest.CiRequest.Image) - //if err != nil { - // log.Println(util.DEVTRON, "Error in digest", err) - // return err - //} - //log.Println(util.DEVTRON, "ExtractDigestUsingPull -> ", digest) + digest, err := helper.ExtractDigestUsingPull(ciCdRequest.CiRequest.Image) + if err != nil { + log.Println(util.DEVTRON, "Error in digest", err) + return err + } + log.Println(util.DEVTRON, "ExtractDigestUsingPull -> ", digest) if len(ciCdRequest.CiRequest.PostCiSteps) > 0 { util.LogStage("running POST-CI steps") // run pre artifact processing @@ -292,18 +293,18 @@ func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { } } - //if ciCdRequest.CiRequest.ScanEnabled { - // util.LogStage("IMAGE SCAN") - // log.Println(util.DEVTRON, " /image-scanner") - // scanEvent := &helper.ScanEvent{Image: ciCdRequest.CiRequest.Image, ImageDigest: digest} - // err = helper.SendEventToClairUtility(scanEvent) - // if err != nil { - // log.Println(err) - // return err - // - // } - // log.Println(util.DEVTRON, " /image-scanner") - //} + if ciCdRequest.CiRequest.ScanEnabled { + util.LogStage("IMAGE SCAN") + log.Println(util.DEVTRON, " /image-scanner") + scanEvent := &helper.ScanEvent{Image: ciCdRequest.CiRequest.Image, ImageDigest: digest} + err = helper.SendEventToClairUtility(scanEvent) + if err != nil { + log.Println(err) + return err + + } + log.Println(util.DEVTRON, " /image-scanner") + } return nil } From fd758d45bed23f9e4d4e912a5134905515803a49 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Sun, 6 Aug 2023 21:43:24 +0530 Subject: [PATCH 09/11] remove code duplicate --- ciStages.go | 68 ++++++++++--------------------------------- helper/EventHelper.go | 1 + 2 files changed, 16 insertions(+), 53 deletions(-) diff --git a/ciStages.go b/ciStages.go index 697cfb79..47aa7a3a 100644 --- a/ciStages.go +++ b/ciStages.go @@ -97,16 +97,17 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e } // git handling log.Println(util.DEVTRON, " git") - err = helper.CloneAndCheckout(ciCdRequest.CiRequest.CiProjectDetails) - if err != nil { - log.Println(util.DEVTRON, "clone err: ", err) - return artifactUploaded, err - } - log.Println(util.DEVTRON, " /git") - - // Start docker daemon - log.Println(util.DEVTRON, " docker-build") + if ciCdRequest.CiRequest.Type != util.WEBHOOK { + err = helper.CloneAndCheckout(ciCdRequest.CiRequest.CiProjectDetails) + if err != nil { + log.Println(util.DEVTRON, "clone err: ", err) + return artifactUploaded, err + } + log.Println(util.DEVTRON, " /git") + // Start docker daemon + log.Println(util.DEVTRON, " docker-build") + } helper.StartDockerDaemon(ciCdRequest.CiRequest.DockerConnection, ciCdRequest.CiRequest.DockerRegistryURL, ciCdRequest.CiRequest.DockerCert, ciCdRequest.CiRequest.DefaultAddressPoolBaseCidr, ciCdRequest.CiRequest.DefaultAddressPoolSize, ciCdRequest.CiRequest.CiBuildDockerMtuValue) scriptEnvs, err := getGlobalEnvVariables(ciCdRequest) if err != nil { @@ -133,18 +134,18 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e refStageMap[ref.Id] = ref.Steps } - var preeCiStageOutVariable map[int]map[string]*helper.VariableObject + var preCiStageOutVariable map[int]map[string]*helper.VariableObject var step *helper.StepObject var preCiDuration float64 start = time.Now() metrics.PreCiStartTime = start - buildSkipEnabled := ciBuildConfigBean != nil && ciBuildConfigBean.CiBuildType == helper.BUILD_SKIP_BUILD_TYPE + buildSkipEnabled := (ciBuildConfigBean != nil && ciBuildConfigBean.CiBuildType == helper.BUILD_SKIP_BUILD_TYPE) || ciCdRequest.CiRequest.Type == util.WEBHOOK if len(ciCdRequest.CiRequest.PreCiSteps) > 0 { if !buildSkipEnabled { util.LogStage("running PRE-CI steps") } // run pre artifact processing - preeCiStageOutVariable, step, err = RunCiCdSteps(STEP_TYPE_PRE, ciCdRequest.CiRequest.PreCiSteps, refStageMap, scriptEnvs, nil) + preCiStageOutVariable, step, err = RunCiCdSteps(STEP_TYPE_PRE, ciCdRequest.CiRequest.PreCiSteps, refStageMap, scriptEnvs, nil) preCiDuration = time.Since(start).Seconds() if err != nil { log.Println(err) @@ -169,7 +170,7 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e // build success will always be false scriptEnvs[util.ENV_VARIABLE_BUILD_SUCCESS] = "false" // run post artifact processing - RunCiCdSteps(STEP_TYPE_POST, postCiStepsToTriggerOnCiFail, refStageMap, scriptEnvs, preeCiStageOutVariable) + RunCiCdSteps(STEP_TYPE_POST, postCiStepsToTriggerOnCiFail, refStageMap, scriptEnvs, preCiStageOutVariable) } // code-block ends return sendFailureNotification(string(Build), ciCdRequest.CiRequest, "", "", metrics, artifactUploaded, err) @@ -184,7 +185,7 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e // sending build success as true always as post-ci triggers only if ci gets success scriptEnvs[util.ENV_VARIABLE_BUILD_SUCCESS] = "true" // run post artifact processing - _, step, err = RunCiCdSteps(STEP_TYPE_POST, ciCdRequest.CiRequest.PostCiSteps, refStageMap, scriptEnvs, preeCiStageOutVariable) + _, step, err = RunCiCdSteps(STEP_TYPE_POST, ciCdRequest.CiRequest.PostCiSteps, refStageMap, scriptEnvs, preCiStageOutVariable) postCiDuration = time.Since(start).Seconds() if err != nil { return sendFailureNotification(string(PostCi)+step.Name, ciCdRequest.CiRequest, "", "", metrics, artifactUploaded, err) @@ -269,45 +270,6 @@ func runCIStages(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, e return artifactUploaded, nil } -func runScanningAndPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent) error { - log.Println(util.DEVTRON, "runScanningAndPostCiSteps", ciCdRequest) - refStageMap := make(map[int][]*helper.StepObject) - for _, ref := range ciCdRequest.CiRequest.RefPlugins { - refStageMap[ref.Id] = ref.Steps - } - helper.StartDockerDaemon(ciCdRequest.CiRequest.DockerConnection, ciCdRequest.CiRequest.DockerRegistryURL, ciCdRequest.CiRequest.DockerCert, ciCdRequest.CiRequest.DefaultAddressPoolBaseCidr, ciCdRequest.CiRequest.DefaultAddressPoolSize, ciCdRequest.CiRequest.CiBuildDockerMtuValue) - log.Println(util.DEVTRON, "ExtractDigestUsingPull", ciCdRequest.CiRequest.Image) - digest, err := helper.ExtractDigestUsingPull(ciCdRequest.CiRequest.Image) - if err != nil { - log.Println(util.DEVTRON, "Error in digest", err) - return err - } - log.Println(util.DEVTRON, "ExtractDigestUsingPull -> ", digest) - if len(ciCdRequest.CiRequest.PostCiSteps) > 0 { - util.LogStage("running POST-CI steps") - // run pre artifact processing - _, _, err := RunCiCdSteps(STEP_TYPE_POST, ciCdRequest.CiRequest.PostCiSteps, refStageMap, nil, nil) - if err != nil { - log.Println(err) - return err - - } - } - if ciCdRequest.CiRequest.ScanEnabled { - util.LogStage("IMAGE SCAN") - log.Println(util.DEVTRON, " /image-scanner") - scanEvent := &helper.ScanEvent{Image: ciCdRequest.CiRequest.Image, ImageDigest: digest} - err = helper.SendEventToClairUtility(scanEvent) - if err != nil { - log.Println(err) - return err - - } - log.Println(util.DEVTRON, " /image-scanner") - } - return nil -} - func getPostCiStepToRunOnCiFail(postCiSteps []*helper.StepObject) []*helper.StepObject { var postCiStepsToTriggerOnCiFail []*helper.StepObject if len(postCiSteps) > 0 { diff --git a/helper/EventHelper.go b/helper/EventHelper.go index 9f92971a..f6316817 100644 --- a/helper/EventHelper.go +++ b/helper/EventHelper.go @@ -142,6 +142,7 @@ type CiRequest struct { ImageRetryCount int `json:"imageRetryCount"` ImageRetryInterval int `json:"imageRetryInterval"` Image string `json:"image"` + Type string `json:"type"` } type CdRequest struct { From b8047ab4ccf3ea6cadf49b84f87ca6578c7b806b Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Sun, 6 Aug 2023 21:51:49 +0530 Subject: [PATCH 10/11] code-refactor --- App.go | 108 --------------------------------------------------------- 1 file changed, 108 deletions(-) diff --git a/App.go b/App.go index 34e23a31..459aaee1 100644 --- a/App.go +++ b/App.go @@ -44,116 +44,10 @@ func handleCleanup(ciCdRequest helper.CiCdTriggerEvent, exitCode *int, source st } func main() { - //args := `{"type":"WEBHOOK","ciRequest":{"workflowNamePrefix":"ci","pipelineName":"","pipelineId":4,"dockerImageTag":"","dockerRegistryId":"","dockerRegistryType":"","dockerRegistryURL":"","dockerConnection":"","dockerCert":"","dockerRepository":"","checkoutPath":"","dockerUsername":"","dockerPassword":"","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"","ciProjectDetails":null,"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":0,"ciImage":"ashexp/test:002a2133-47-115","namespace":"devtron-ci","workflowId":0,"triggeredBy":4,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","ciArtifactBucket":"","ciArtifactFileName":"","ciArtifactRegion":"","scanEnabled":true,"cloudProvider":"S3","blobStorageConfigured":false,"blobStorageS3Config":null,"azureBlobConfig":null,"gcpBlobConfig":null,"blobStorageLogsKey":"arsenal-v1/ci","inAppLoggingEnabled":false,"defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":null,"postCiSteps":[{"name":"Task1","index":1,"stepType":"INLINE","executorType":"SHELL","script":"#!/bin/shset -eo pipefail #set -v ## uncomment this to debug the scriptecho \"HelloWorld\"","inputVars":null,"exposedPorts":null,"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMount":null,"sourceCodeMount":null,"extraVolumeMounts":null,"artifactPaths":null,"triggerIfParentStageFail":false}],"refPlugins":null,"appName":"","triggerByAuthor":"","ciBuildConfig":null,"ciBuildDockerMtuValue":-1,"ignoreDockerCachePush":false,"ignoreDockerCachePull":false,"cacheInvalidate":false,"IsPvcMounted":false,"extraEnvironmentVariables":null,"enableBuildContext":false,"appId":55,"environmentId":0,"orchestratorHost":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","orchestratorToken":"","isExtRun":false,"imageRetryCount":0,"imageRetryInterval":5,"image":"ashexp/test:002a2133-47-115"},"cdRequest":null}` - // args := `'{"name":"ci","inputs":{},"outputs":{},"metadata":{},"container":{"name":"","image":"quay.io/devtron/test:75949ec6-890-6954","ports":[{"name":"app-data","containerPort":9102}],"env":[{"name":"IMAGE_SCANNER_ENDPOINT","value":"http://image-scanner-service.devtroncd:80"},{"name":"CI_CD_EVENT","value":"{\"type\":\"WEBHOOK\",\"ciRequest\":{\"workflowNamePrefix\":\"ci\",\"pipelineName\":\"\",\"pipelineId\":4,\"dockerImageTag\":\"\",\"dockerRegistryId\":\"\",\"dockerRegistryType\":\"\",\"dockerRegistryURL\":\"\",\"dockerConnection\":\"\",\"dockerCert\":\"\",\"dockerRepository\":\"\",\"checkoutPath\":\"\",\"dockerUsername\":\"\",\"dockerPassword\":\"\",\"awsRegion\":\"\",\"accessKey\":\"\",\"secretKey\":\"\",\"ciCacheLocation\":\"\",\"ciCacheRegion\":\"\",\"ciCacheFileName\":\"\",\"ciProjectDetails\":null,\"containerResources\":{\"minCpu\":\"\",\"maxCpu\":\"\",\"minStorage\":\"\",\"maxStorage\":\"\",\"minEphStorage\":\"\",\"maxEphStorage\":\"\",\"minMem\":\"\",\"maxMem\":\"\"},\"activeDeadlineSeconds\":0,\"ciImage\":\"quay.io/devtron/test:75949ec6-890-6954\",\"namespace\":\"devtron-ci\",\"workflowId\":0,\"triggeredBy\":4,\"cacheLimit\":5000000000,\"beforeDockerBuildScripts\":null,\"afterDockerBuildScripts\":null,\"ciArtifactLocation\":\"\",\"ciArtifactBucket\":\"\",\"ciArtifactFileName\":\"\",\"ciArtifactRegion\":\"\",\"scanEnabled\":true,\"cloudProvider\":\"\",\"blobStorageConfigured\":false,\"blobStorageS3Config\":null,\"azureBlobConfig\":null,\"gcpBlobConfig\":null,\"blobStorageLogsKey\":\"devtron/ci\",\"inAppLoggingEnabled\":true,\"defaultAddressPoolBaseCidr\":\"\",\"defaultAddressPoolSize\":0,\"preCiSteps\":null,\"postCiSteps\":[{\"name\":\"Task - // 1\",\"index\":1,\"stepType\":\"INLINE\",\"executorType\":\"SHELL\",\"script\":\"#!/bin/sh - // \set -eo pipefail \#set -v ## uncomment this to debug the script - // \\echo \\\"Hello - // World\\\"\",\"inputVars\":null,\"exposedPorts\":null,\"outputVars\":null,\"triggerSkipConditions\":null,\"successFailureConditions\":null,\"dockerImage\":\"\",\"command\":\"\",\"args\":null,\"customScriptMount\":null,\"sourceCodeMount\":null,\"extraVolumeMounts\":null,\"artifactPaths\":null,\"triggerIfParentStageFail\":false}],\"refPlugins\":null,\"appName\":\"\",\"triggerByAuthor\":\"\",\"ciBuildConfig\":null,\"ciBuildDockerMtuValue\":-1,\"ignoreDockerCachePush\":false,\"ignoreDockerCachePull\":false,\"cacheInvalidate\":false,\"IsPvcMounted\":false,\"extraEnvironmentVariables\":null,\"enableBuildContext\":true,\"appId\":55,\"environmentId\":0,\"orchestratorHost\":\"http://20.231.45.51:32267/orchestrator/webhook/msg/nats\",\"orchestratorToken\":\"cCtrbHF2ditXTDZ5UEkza1dVTXUyZXcrUWljPQo\",\"isExtRun\":false,\"imageRetryCount\":0,\"imageRetryInterval\":5,\"image\":\"quay.io/devtron/test:75949ec6-890-6954\"},\"cdRequest\":null}"},{"name":"IN_APP_LOGGING","value":"true"}],"resources":{"limits":{"cpu":"500m","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}},"securityContext":{"privileged":true}},"archiveLocation":{"archiveLogs":false},"activeDeadlineSeconds":2147483632}'` //args := `{"type":"CI","ciRequest":{"DockerBuildTargetPlatform":"linux/arm64", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/getting-started-nodejs","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"da3ba3254712965b5944a6271e71bff91fe51f20","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` //args := `{"type":"DryRun","dryRunRequest":{"buildPackParams":{"builderId":"gcr.io/buildpacks/builder:v1"},"DockerBuildTargetPlatform":"", "workflowNamePrefix":"16-ci-25-w5x1-70","pipelineName":"ci-25-w5x1","pipelineId":70,"dockerImageTag":"da3ba326-70-17","dockerRegistryId":"devtron-quay","dockerRegistryType":"other","dockerRegistryURL":"https://quay.io/devtron","dockerConnection":"secure","dockerCert":"","dockerBuildArgs":"{}","dockerRepository":"test","dockerfileLocation":"Dockerfile","dockerUsername":"devtron+devtest","dockerPassword":"5WEDXDJMP6RV1CG1KKFJQL3MQOLC64JKM6K684WPEBKVWKOZ4LSMBHEHJU1HBGXK","awsRegion":"","accessKey":"","secretKey":"","ciCacheLocation":"","ciCacheRegion":"","ciCacheFileName":"ci-25-w5x1-70.tar.gz","ciProjectDetails":[{"gitRepository":"https://github.com/devtron-labs/sample-go-app","materialName":"1-getting-started-nodejs","checkoutPath":"./","fetchSubmodules":false,"commitHash":"8654623ec2bd9efd663935cb8332c8c765541837","gitTag":"","commitTime":"2022-04-12T20:26:08+05:30","type":"SOURCE_TYPE_BRANCH_FIXED","message":"Update README.md","author":"Prakarsh \u003c71125043+prakarsh-dt@users.noreply.github.com\u003e","gitOptions":{"userName":"","password":"","sshPrivateKey":"","accessToken":"","authMode":"ANONYMOUS"},"sourceType":"SOURCE_TYPE_BRANCH_FIXED","sourceValue":"master","WebhookData":{"Id":0,"EventActionType":"","Data":null}}],"containerResources":{"minCpu":"","maxCpu":"","minStorage":"","maxStorage":"","minEphStorage":"","maxEphStorage":"","minMem":"","maxMem":""},"activeDeadlineSeconds":3600,"ciImage":"quay.io/devtron/ci-runner:1290cf23-182-8015","namespace":"devtron-ci","workflowId":16,"triggeredBy":8,"cacheLimit":5000000000,"beforeDockerBuildScripts":null,"afterDockerBuildScripts":null,"ciArtifactLocation":"","invalidateCache":true,"scanEnabled":false,"cloudProvider":"AZURE","azureBlobConfig":{"enabled":true,"accountName":"devtrondemoblob","blobContainerCiLog":"","blobContainerCiCache":"cache","accountKey":"y1/K13YMp/v7uuvZNkKJ4dS3CyGc37bPIN9Hv8MVhog6OkG0joV05proQReMQIJQ8qXp0JVpj+mz+AStHNKR3Q=="},"minioEndpoint":"","defaultAddressPoolBaseCidr":"","defaultAddressPoolSize":0,"preCiSteps":[{"name":"Task 1","index":1,"stepType":"INLINE","executorType":"SHELL","refPluginId":0,"script":"echo $","inputVars":null,"exposedPorts":{"0":0},"outputVars":null,"triggerSkipConditions":null,"successFailureConditions":null,"dockerImage":"","command":"","args":null,"customScriptMountDestinationPath":{"sourcePath":"","destinationPath":""},"sourceCodeMountDestinationPath":{"sourcePath":"","destinationPath":""},"extraVolumeMounts":null,"artifactPaths":null}],"postCiSteps":null,"refPlugins":null},"cdRequest":null}` //' {"workflowNamePrefix":"55-suraj-23-ci-suraj-test-pipeline-8","pipelineName":"suraj-23-ci-suraj-test-pipeline","pipelineId":8,"dockerImageTag":"a6b809c4be87c217feba4af15cf5ebc3cafe21e0","dockerRegistryURL":"686244538589.dkr.ecr.us-east-2.amazonaws.com","dockerRepository":"test/suraj-23","dockerfileLocation":"./notifier/Dockerfile","awsRegion":"us-east-2","ciCacheLocation":"ci-caching","ciCacheFileName":"suraj-23-ci-suraj-test-pipeline.tar.gz","ciProjectDetails":[{"gitRepository":"https://gitlab.com/devtron/notifier.git","materialName":"1-notifier","checkoutPath":"./notifier","commitHash":"d4df38bcd065004014d255c2203d592a91585955","commitTime":"0001-01-01T00:00:00Z","branch":"ci_with_argo","type":"SOURCE_TYPE_BRANCH_FIXED","message":"test-commit","gitOptions":{"userName":"Suraj24","password":"Devtron@1234","sshKey":"","accessToken":"","authMode":"USERNAME_PASSWORD"}},{"gitRepository":"https://gitlab.com/devtron/orchestrator.git","materialName":"2-orchestrator","checkoutPath":"./orch","commitHash":"","commitTime":"0001-01-01T00:00:00Z","branch":"ci_with_argo","type":"SOURCE_TYPE_BRANCH_FIXED","message":"","gitOptions":{"userName":"Suraj24","password":"Devtron@1234","sshKey":"","accessToken":"","authMode":""}}],"ciImage":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:latest","namespace":"default"}' - // args := `{ - // "type": "WEBHOOK", - // "ciRequest": { - // "workflowNamePrefix": "ci", - // "pipelineName": "", - // "pipelineId": 4, - // "dockerImageTag": "", - // "dockerRegistryId": "", - // "dockerRegistryType": "", - // "dockerRegistryURL": "", - // "dockerConnection": "", - // "dockerCert": "", - // "dockerRepository": "", - // "checkoutPath": "", - // "dockerUsername": "", - // "dockerPassword": "", - // "awsRegion": "", - // "accessKey": "", - // "secretKey": "", - // "ciCacheLocation": "", - // "ciCacheRegion": "", - // "ciCacheFileName": "", - // "ciProjectDetails": null, - // "containerResources": { - // "minCpu": "", - // "maxCpu": "", - // "minStorage": "", - // "maxStorage": "", - // "minEphStorage": "", - // "maxEphStorage": "", - // "minMem": "", - // "maxMem": "" - // }, - // "activeDeadlineSeconds": 0, - // "ciImage": "ashexp/test:002a2133-47-115", - // "namespace": "devtron-ci", - // "workflowId": 0, - // "triggeredBy": 4, - // "cacheLimit": 5000000000, - // "beforeDockerBuildScripts": null, - // "afterDockerBuildScripts": null, - // "ciArtifactLocation": "", - // "ciArtifactBucket": "", - // "ciArtifactFileName": "", - // "ciArtifactRegion": "", - // "scanEnabled": true, - // "cloudProvider": "", - // "blobStorageConfigured": false, - // "blobStorageS3Config": null, - // "azureBlobConfig": null, - // "gcpBlobConfig": null, - // "blobStorageLogsKey": "devtron/ci", - // "inAppLoggingEnabled": true, - // "defaultAddressPoolBaseCidr": "", - // "defaultAddressPoolSize": 0, - // "preCiSteps": null, - // "postCiSteps": [ - // { - // "name": "Task1", - // "index": 1, - // "stepType": "INLINE", - // "executorType": "SHELL", - // "script": "#!/bin/shset -eo pipefail #set -v ## uncomment this to debug the script echo ", - // "inputVars": null, - // "exposedPorts": null, - // "outputVars": null, - // "triggerSkipConditions": null, - // "successFailureConditions": null, - // "dockerImage": "", - // "command": "", - // "args": null, - // "customScriptMount": null, - // "sourceCodeMount": null, - // "extraVolumeMounts": null, - // "artifactPaths": null, - // "triggerIfParentStageFail": false - // } - // ], - // "refPlugins": null, - // "appName": "", - // "triggerByAuthor": "", - // "ciBuildConfig": null, - // "ciBuildDockerMtuValue": -1, - // "ignoreDockerCachePush": false, - // "ignoreDockerCachePull": false, - // "cacheInvalidate": false, - // "IsPvcMounted": false, - // "extraEnvironmentVariables": null, - // "enableBuildContext": true, - // "appId": 55, - // "environmentId": 0, - // "orchestratorHost": "http://20.231.45.51:32267/orchestrator/webhook/msg/nats", - // "orchestratorToken": "cCtrbHF2ditXTDZ5UEkza1dVTXUyZXcrUWljPQo", - // "isExtRun": false, - // "imageRetryCount": 0, - // "imageRetryInterval": 5, - // "image": "ashexp/test:002a2133-47-115" - // }, - // "cdRequest": null - //}` LoggingMode := "RunMode" if len(os.Args) > 1 { LoggingMode = os.Args[1] @@ -199,8 +93,6 @@ func processEvent(args string) { defer handleCleanup(*ciCdRequest, &exitCode, util.Source_Defer) if ciCdRequest.Type == util.CIEVENT { HandleCIEvent(ciCdRequest, &exitCode) - } else if ciCdRequest.Type == util.WEBHOOK { - runScanningAndPostCiSteps(ciCdRequest) } else { HandleCDEvent(ciCdRequest, &exitCode) } From dd5f1f7b067f8ba21b80218a4ca2926cc61274da Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Sun, 6 Aug 2023 21:53:36 +0530 Subject: [PATCH 11/11] code-refactor --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d31ab08d..79656d1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /go/bin/cirunner FROM docker:20.10.24-dind # All these steps will be cached #RUN apk add --no-cache ca-certificates -RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && apk add --no-cache --virtual openssh +RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && apk add openssh RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN apk -Uuv add groff less python3 py3-pip RUN pip3 install awscli