diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml index 5f3970f6e8..7a54c0ec57 100644 --- a/.semaphore/cp_dockerfile_build.yml +++ b/.semaphore/cp_dockerfile_build.yml @@ -1,3 +1,5 @@ +# $schema: https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml +# yaml-language-server: $schema=https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml # This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common # template and configurations in service.yml. # Any modifications made to ths file will be overwritten by the generated content in nightly runs. @@ -26,7 +28,6 @@ global_job_config: - checkout - if [[ $SEMAPHORE_GIT_BRANCH =~ ^7\..* ]]; then sem-version java 8; else sem-version java 17; fi - sem-version python 3.14 - - . vault-setup - . cache-maven restore - pip install tox==3.28.0 - export GIT_COMMIT=$(git rev-parse --verify HEAD --short) @@ -60,8 +61,13 @@ global_job_config: else export PLATFORM_LABEL="" fi - - export PACKAGING_BUILD_ARGS=" -DCONFLUENT_VERSION=$CONFLUENT_VERSION -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION - -DALLOW_UNSIGNED=$ALLOW_UNSIGNED" + # Guard -D flags: fabric8 fails when build args resolve to null from empty -D values (e.g. -DCONFLUENT_VERSION=) + - | + export PACKAGING_BUILD_ARGS="" + if [[ -n "$CONFLUENT_VERSION" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_VERSION=$CONFLUENT_VERSION"; fi + if [[ -n "$PLATFORM_LABEL" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL"; fi + if [[ -n "$CONFLUENT_DEB_VERSION" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION"; fi + if [[ -n "$ALLOW_UNSIGNED" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DALLOW_UNSIGNED=$ALLOW_UNSIGNED"; fi - >- if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then if [[ $IS_RC ]]; then @@ -71,27 +77,29 @@ global_job_config: fi fi # Overwrite maven global configuration - . vault-sem-get-secret maven-settings-cp-dockerfile + gh api repos/confluentinc/depot/contents/python/cp_devtools/templates/template_resources/maven/maven_semaphore_cp_dockerfile_settings.xml | jq -r '.content' | base64 -d > template-settings.xml + [[ -f ~/.m2/settings.xml ]] && mv ~/.m2/settings.xml ~/.m2/settings.xml.bak; envsubst < template-settings.xml > ~/.m2/settings.xml else echo "This job is not a isHotfixJob or isRcJob (What we know how to handle) - and we don't know how to handle it" fi - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/" - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY - - export LATEST_TAG=8.2.x-latest + - export LATEST_TAG=$BRANCH_TAG-latest - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" - export DOCKER_REPOS="confluentinc/cp-base-java confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" - export COMMUNITY_DOCKER_REPOS="" + - export S390X_DOCKER_REPOS="confluentinc/cp-base-java" + - export S390X_MAVEN_MODULES="base-java" - | + export COMMUNITY_MVN_PL_ARGS="" + export S390X_MVN_PL_ARGS="-pl ${S390X_MAVEN_MODULES// /,} -am" if [[ $SKIP_COMMUNITY == "True" ]]; then # Filter out community repos from DOCKER_REPOS DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs) export DOCKER_REPOS echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS" - # Set Maven arguments for skipping community modules - export MAVEN_EXTRA_ARGS="" - # Check if current DOCKER_IMAGE is in community repos, skip job execution for skip_repo in $COMMUNITY_DOCKER_REPOS; do if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then @@ -100,12 +108,15 @@ global_job_config: return 130 fi done - else - export MAVEN_EXTRA_ARGS="" + + S390X_DOCKER_REPOS=$(comm -23 <(echo "$S390X_DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs) + export S390X_DOCKER_REPOS + echo "S390X_DOCKER_REPOS after skipping community images - $S390X_DOCKER_REPOS" fi - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 + - export S390X_ARCH=.s390x blocks: - name: Validation dependencies: [] @@ -115,12 +126,12 @@ blocks: jobs: - name: Validation commands: - - . sem-pint -c + - ci-sem-pint -c - name: Build, Test, & Scan AMD dependencies: ["Validation"] run: # don't run the tests on non-functional changes... - when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/'], default_branch: 'master'})" + when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/', 'service.yml', 'README.md'], default_branch: 'master'})" task: jobs: - name: Build, Test, & Scan ubi9 @@ -128,12 +139,12 @@ blocks: - export OS_TAG="-ubi9" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH - - ci-tools ci-update-version + - ci-tools ci-update-version --direct-pom-edit - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $COMMUNITY_MVN_PL_ARGS - . cache-maven store - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; @@ -271,7 +282,7 @@ blocks: dependencies: ["Validation"] run: # don't run the tests on non-functional changes... - when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/'], default_branch: 'master'})" + when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/', 'service.yml', 'README.md'], default_branch: 'master'})" task: agent: machine: @@ -284,10 +295,10 @@ blocks: - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - ci-tools ci-update-version - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + - ci-tools ci-update-version --direct-pom-edit + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $COMMUNITY_MVN_PL_ARGS - . cache-maven store - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: @@ -428,9 +439,65 @@ blocks: - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$ARM_ARCH - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - name: Build & Test s390x + dependencies: ["Validation"] + run: + # don't run the tests on non-functional changes... + when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/', 'service.yml', 'README.md'], default_branch: 'master'})" + task: + jobs: + - name: Build & Test s390x ubi9 + commands: + - export OS_TAG="-ubi9" + - ci-tools ci-update-version --direct-pom-edit + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + # Register QEMU binfmt for s390x so Docker can run s390x containers on amd64 for testing. + # ARM builds run on native arm64 machines, but s390x cross-compiles on amd64 via BuildX and needs QEMU. + - docker run --privileged --rm tonistiigi/binfmt --install s390x + - 'docker buildx ls | grep -q s390x || (echo "ERROR: s390x binfmt registration failed" && exit 1)' + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean install dependency:analyze validate -U $S390X_MVN_PL_ARGS -Ddocker.registry=$DOCKER_DEV_REGISTRY + -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.buildx.platforms=linux/s390x $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true + - . cache-maven store + - export S390X_DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${S390X_DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export S390X_DOCKER_DEV_FULL_IMAGES=${S390X_DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH + - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results + - name: Deploy s390x confluentinc/cp-base-java + dependencies: ["Build & Test s390x"] + run: + when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" + task: + jobs: + - name: Deploy s390x confluentinc/cp-base-java ubi9 + commands: + - export OS_TAG="-ubi9" + - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/cp-base-java + - export GIT_COMMIT_TAG=$GIT_COMMIT$OS_TAG$S390X_ARCH + - export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG$S390X_ARCH + - export DEV_IMAGE_FULL=${DOCKER_DEV_REGISTRY}confluentinc/cp-base-java:$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + - docker pull $DEV_IMAGE_FULL + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - docker push $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - docker push $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - sign-images $PROD_IMAGE_NAME:$GIT_COMMIT_TAG + - sign-images $PROD_IMAGE_NAME:$BRANCH_BUILD_TAG + - export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$PACKAGE_TAG + - docker push $PROD_IMAGE_NAME:$PACKAGE_TAG + - sign-images $PROD_IMAGE_NAME:$PACKAGE_TAG + - export LATEST_PUSH_TAG=$LATEST_TAG$OS_TAG$S390X_ARCH + - docker tag $DEV_IMAGE_FULL $PROD_IMAGE_NAME:$LATEST_PUSH_TAG + - docker push $PROD_IMAGE_NAME:$LATEST_PUSH_TAG - name: Create Manifest and Maven Deploy dependencies: ["Deploy AMD confluentinc/cp-base-java", "Deploy AMD confluentinc/cp-base-new", "Deploy AMD confluentinc/cp-base-lite", "Deploy AMD confluentinc/cp-jmxterm", "Deploy ARM confluentinc/cp-base-java", - "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite", "Deploy ARM confluentinc/cp-jmxterm"] + "Deploy ARM confluentinc/cp-base-new", "Deploy ARM confluentinc/cp-base-lite", "Deploy ARM confluentinc/cp-jmxterm", "Deploy s390x confluentinc/cp-base-java"] run: when: "branch = 'master' or branch =~ '^[0-9]+\\.[0-9]+\\.x$' or branch =~ '^[0-9]+\\.[0-9]+\\.[0-9]+(-cp[0-9]+)?(-rc[0-9]+)?$'" task: @@ -438,32 +505,36 @@ blocks: - name: Create Manifest and Maven Deploy commands: - export DOCKER_PROD_IMAGE_NAME=$DOCKER_PROD_REGISTRY${DOCKER_REPOS// / $DOCKER_PROD_REGISTRY} - - ci-tools ci-update-version + - ci-tools ci-update-version --direct-pom-edit - ci-tools ci-push-tag - |- if [[ ! $IS_RELEASE && ! $IS_PREVIEW ]]; then - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/ -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests -Ddocker.skip-build=true -Ddocker.skip-test=true $MAVEN_EXTRA_ARGS + mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/ -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests -Ddocker.skip-build=true -Ddocker.skip-test=true $COMMUNITY_MVN_PL_ARGS fi # Create manifest - >- for image in $DOCKER_PROD_IMAGE_NAME; do export OS_TAG="-ubi9" + # Check if current image has an s390x build. ${IS_S390X:+...} conditionally + # includes the s390x digest in the manifest only for repos in S390X_DOCKER_REPOS. + IS_S390X="" + for s390x_repo in $S390X_DOCKER_REPOS; do if [[ "$image" == "$DOCKER_PROD_REGISTRY$s390x_repo" ]]; then IS_S390X="true"; break; fi; done export GIT_TAG=$GIT_COMMIT$OS_TAG - docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH + docker manifest create $image:$GIT_TAG $image:$GIT_TAG$AMD_ARCH $image:$GIT_TAG$ARM_ARCH ${IS_S390X:+$image:$GIT_TAG$S390X_ARCH} docker manifest push $image:$GIT_TAG docker pull $image:$GIT_TAG sign-images $image:$GIT_TAG export BRANCH_BUILD_TAG=$BRANCH_TAG-$BUILD_NUMBER$OS_TAG - docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH + docker manifest create $image:$BRANCH_BUILD_TAG $image:$BRANCH_BUILD_TAG$AMD_ARCH $image:$BRANCH_BUILD_TAG$ARM_ARCH ${IS_S390X:+$image:$BRANCH_BUILD_TAG$S390X_ARCH} docker manifest push $image:$BRANCH_BUILD_TAG docker pull $image:$BRANCH_BUILD_TAG sign-images $image:$BRANCH_BUILD_TAG export PACKAGE_TAG=$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG - docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH + docker manifest create $image:$PACKAGE_TAG $image:$PACKAGE_TAG$AMD_ARCH $image:$PACKAGE_TAG$ARM_ARCH ${IS_S390X:+$image:$PACKAGE_TAG$S390X_ARCH} docker manifest push $image:$PACKAGE_TAG export LATEST_MANIFEST_TAG=$LATEST_TAG$OS_TAG - docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH + docker manifest create $image:$LATEST_MANIFEST_TAG $image:$LATEST_MANIFEST_TAG$AMD_ARCH $image:$LATEST_MANIFEST_TAG$ARM_ARCH ${IS_S390X:+$image:$LATEST_MANIFEST_TAG$S390X_ARCH} docker manifest push $image:$LATEST_MANIFEST_TAG done after_pipeline: diff --git a/.semaphore/cp_dockerfile_promote.yml b/.semaphore/cp_dockerfile_promote.yml index 330c2ff951..ba00c89a24 100644 --- a/.semaphore/cp_dockerfile_promote.yml +++ b/.semaphore/cp_dockerfile_promote.yml @@ -1,3 +1,5 @@ +# $schema: https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml +# yaml-language-server: $schema=https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml # This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common # template and configurations in service.yml. # Any modifications made to ths file will be overwritten by the generated content in nightly runs. @@ -46,6 +48,7 @@ global_job_config: - docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 + - export S390X_ARCH=.s390x - export COMMUNITY_DOCKER_REPOS="" - | if [[ $SKIP_COMMUNITY == "True" ]]; then @@ -57,8 +60,8 @@ global_job_config: fi done fi - blocks: + - name: Promote AMD dependencies: [] task: @@ -187,6 +190,7 @@ blocks: docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest-$OS_TYPE$AMD_ARCH docker push $DOCKER_REPO:latest-$OS_TYPE$AMD_ARCH fi + - name: Promote ARM dependencies: [] task: @@ -315,8 +319,44 @@ blocks: docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest-$OS_TYPE$ARM_ARCH docker push $DOCKER_REPO:latest-$OS_TYPE$ARM_ARCH fi + + - name: Promote s390x + dependencies: [] + task: + jobs: + - name: Promote confluentinc/cp-base-java ubi9 s390x + env_vars: + - name: DOCKER_IMAGE + value: confluentinc/cp-base-java + commands: + - export OS_TYPE="ubi9" + - export DOCKER_REPO="confluentinc/cp-base-java" + - if [[ ! "$OS_TYPE" ]]; then export OS_TAG=""; elif [[ "$OS_TYPE" =~ $PROMOTE_OS_TYPE* ]]; then export OS_TAG="-$OS_TYPE"; fi + - export INTERNAL_IMAGE_TAG="$BRANCH_TAG-$PACKAGING_BUILD_NUMBER$OS_TAG$S390X_ARCH" + - export PROMOTED_TAG="$PROMOTED_TAG_PREFIX$OS_TAG$S390X_ARCH" + - docker pull $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG + - docker tag $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG $DOCKER_REPO:$PROMOTED_TAG + - docker run --rm $DOCKER_REPO:$PROMOTED_TAG sh -c "grep staging /etc/yum.repos.d/confluent.repo || grep staging /etc/apt/sources.list" || export STAGING_CHECK_SUCCEED="true" + - if [[ ! "$STAGING_CHECK_SUCCEED" ]]; then echo "Detected there was a staging repo in image $DOCKER_REPO:$PROMOTED_TAG refusing to promote." && exit 1; fi + - docker push $DOCKER_REPO:$PROMOTED_TAG + - >- + if [[ ! "$OS_TYPE" ]] || [[ "$OS_TYPE" =~ ubi* ]]; then + export APPLY_TAG=$CONFLUENT_VERSION$S390X_ARCH + docker tag $DOCKER_PROD_REGISTRY$DOCKER_REPO:$INTERNAL_IMAGE_TAG $DOCKER_REPO:$APPLY_TAG + docker push $DOCKER_REPO:$APPLY_TAG + export APPLIED="true" + fi + - >- + if [[ $UPDATE_LATEST_TAG == "True" ]]; then + if [[ $APPLIED ]]; then + docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest$S390X_ARCH + docker push $DOCKER_REPO:latest$S390X_ARCH + fi + docker tag $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:latest-$OS_TYPE$S390X_ARCH + docker push $DOCKER_REPO:latest-$OS_TYPE$S390X_ARCH + fi - name: Create Manifest - dependencies: ["Promote AMD", "Promote ARM"] + dependencies: ["Promote AMD", "Promote ARM", "Promote s390x"] task: jobs: - name: Create Manifest confluentinc/cp-base-java ubi9 @@ -328,21 +368,21 @@ blocks: - if [[ ! "$OS_TYPE" ]]; then export OS_TAG=""; elif [[ "$OS_TYPE" =~ $PROMOTE_OS_TYPE* ]]; then export OS_TAG="-$OS_TYPE"; fi - export DOCKER_REPO="confluentinc/cp-base-java" - export PROMOTED_TAG="$PROMOTED_TAG_PREFIX$OS_TAG" - - docker manifest create $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:$PROMOTED_TAG$AMD_ARCH $DOCKER_REPO:$PROMOTED_TAG$ARM_ARCH + - docker manifest create $DOCKER_REPO:$PROMOTED_TAG $DOCKER_REPO:$PROMOTED_TAG$AMD_ARCH $DOCKER_REPO:$PROMOTED_TAG$ARM_ARCH $DOCKER_REPO:$PROMOTED_TAG$S390X_ARCH - docker manifest push $DOCKER_REPO:$PROMOTED_TAG - >- if [[ ! "$OS_TYPE" ]] || [[ "$OS_TYPE" =~ ubi* ]]; then - docker manifest create $DOCKER_REPO:$CONFLUENT_VERSION $DOCKER_REPO:$CONFLUENT_VERSION$AMD_ARCH $DOCKER_REPO:$CONFLUENT_VERSION$ARM_ARCH + docker manifest create $DOCKER_REPO:$CONFLUENT_VERSION $DOCKER_REPO:$CONFLUENT_VERSION$AMD_ARCH $DOCKER_REPO:$CONFLUENT_VERSION$ARM_ARCH $DOCKER_REPO:$CONFLUENT_VERSION$S390X_ARCH docker manifest push $DOCKER_REPO:$CONFLUENT_VERSION export APPLIED="true" fi - >- if [[ $UPDATE_LATEST_TAG == "True" ]]; then if [[ $APPLIED ]]; then - docker manifest create $DOCKER_REPO:latest $DOCKER_REPO:latest$AMD_ARCH $DOCKER_REPO:latest$ARM_ARCH + docker manifest create $DOCKER_REPO:latest $DOCKER_REPO:latest$AMD_ARCH $DOCKER_REPO:latest$ARM_ARCH $DOCKER_REPO:latest$S390X_ARCH docker manifest push $DOCKER_REPO:latest fi - docker manifest create $DOCKER_REPO:latest-$OS_TYPE $DOCKER_REPO:latest-$OS_TYPE$AMD_ARCH $DOCKER_REPO:latest-$OS_TYPE$ARM_ARCH + docker manifest create $DOCKER_REPO:latest-$OS_TYPE $DOCKER_REPO:latest-$OS_TYPE$AMD_ARCH $DOCKER_REPO:latest-$OS_TYPE$ARM_ARCH $DOCKER_REPO:latest-$OS_TYPE$S390X_ARCH docker manifest push $DOCKER_REPO:latest-$OS_TYPE fi - name: Create Manifest confluentinc/cp-base-new ubi9 diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9c0812432d..66daa1ba6b 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -1,3 +1,5 @@ +# $schema: https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml +# yaml-language-server: $schema=https://raw.githubusercontent.com/semaphoreio/semaphore/98bb4752c79049d36324e8c76451cd704c7924f3/plumber/spec/priv/v1.0.yml # This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common # template and configurations in service.yml. # Any modifications made to ths file will be overwritten by the generated content in nightly runs. @@ -14,7 +16,7 @@ fail_fast: when: "true" execution_time_limit: - hours: 1 + hours: 2 queue: - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.[0-9]+'" @@ -26,7 +28,6 @@ global_job_config: - checkout - if [[ $SEMAPHORE_GIT_BRANCH =~ ^7\..* ]]; then sem-version java 8; else sem-version java 17; fi - sem-version python 3.14 - - . vault-setup - . cache-maven restore - pip install tox==3.28.0 - export GIT_COMMIT=$(git rev-parse --verify HEAD --short) @@ -53,8 +54,13 @@ global_job_config: else export PLATFORM_LABEL="" fi - - export PACKAGING_BUILD_ARGS=" -DCONFLUENT_VERSION=$CONFLUENT_VERSION -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION - -DALLOW_UNSIGNED=$ALLOW_UNSIGNED" + # Guard -D flags: fabric8 fails when build args resolve to null from empty -D values (e.g. -DCONFLUENT_VERSION=) + - | + export PACKAGING_BUILD_ARGS="" + if [[ -n "$CONFLUENT_VERSION" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_VERSION=$CONFLUENT_VERSION"; fi + if [[ -n "$PLATFORM_LABEL" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PLATFORM_LABEL=$PLATFORM_LABEL"; fi + if [[ -n "$CONFLUENT_DEB_VERSION" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_DEB_VERSION=$CONFLUENT_DEB_VERSION"; fi + if [[ -n "$ALLOW_UNSIGNED" ]]; then PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DALLOW_UNSIGNED=$ALLOW_UNSIGNED"; fi - >- if [[ $IS_RELEASE && $PACKAGING_BUILD_NUMBER ]]; then if [[ $IS_RC ]]; then @@ -64,27 +70,29 @@ global_job_config: fi fi # Overwrite maven global configuration - . vault-sem-get-secret maven-settings-cp-dockerfile + gh api repos/confluentinc/depot/contents/python/cp_devtools/templates/template_resources/maven/maven_semaphore_cp_dockerfile_settings.xml | jq -r '.content' | base64 -d > template-settings.xml + [[ -f ~/.m2/settings.xml ]] && mv ~/.m2/settings.xml ~/.m2/settings.xml.bak; envsubst < template-settings.xml > ~/.m2/settings.xml else echo "This job is not a isHotfixJob or isRcJob (What we know how to handle) - and we don't know how to handle it" fi - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/" - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY - - export LATEST_TAG=8.2.x-latest + - export LATEST_TAG=$BRANCH_TAG-latest - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" - export DOCKER_REPOS="confluentinc/cp-base-java confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" - export COMMUNITY_DOCKER_REPOS="" + - export S390X_DOCKER_REPOS="confluentinc/cp-base-java" + - export S390X_MAVEN_MODULES="base-java" - | + export COMMUNITY_MVN_PL_ARGS="" + export S390X_MVN_PL_ARGS="-pl ${S390X_MAVEN_MODULES// /,} -am" if [[ $SKIP_COMMUNITY == "True" ]]; then # Filter out community repos from DOCKER_REPOS DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs) export DOCKER_REPOS echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS" - # Set Maven arguments for skipping community modules - export MAVEN_EXTRA_ARGS="" - # Check if current DOCKER_IMAGE is in community repos, skip job execution for skip_repo in $COMMUNITY_DOCKER_REPOS; do if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then @@ -93,12 +101,15 @@ global_job_config: return 130 fi done - else - export MAVEN_EXTRA_ARGS="" + + S390X_DOCKER_REPOS=$(comm -23 <(echo "$S390X_DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs) + export S390X_DOCKER_REPOS + echo "S390X_DOCKER_REPOS after skipping community images - $S390X_DOCKER_REPOS" fi - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 + - export S390X_ARCH=.s390x blocks: - name: Validation dependencies: [] @@ -108,7 +119,7 @@ blocks: jobs: - name: Validation commands: - - . sem-pint -c + - ci-sem-pint -c - name: Build, Test, & Scan AMD dependencies: ["Validation"] run: @@ -120,12 +131,12 @@ blocks: - export OS_TAG="-ubi9" - export DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG - export AMD_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$AMD_ARCH }$AMD_ARCH - - ci-tools ci-update-version + - ci-tools ci-update-version --direct-pom-edit - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $COMMUNITY_MVN_PL_ARGS - . cache-maven store - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; @@ -155,10 +166,10 @@ blocks: - export ARM_DOCKER_DEV_FULL_IMAGES=${DOCKER_DEV_FULL_IMAGES// /$ARM_ARCH }$ARM_ARCH - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - - ci-tools ci-update-version - - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY + - ci-tools ci-update-version --direct-pom-edit + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH - -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi9 $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $COMMUNITY_MVN_PL_ARGS - . cache-maven store - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: @@ -167,6 +178,34 @@ blocks: - . publish-test-results - artifact push workflow target/test-results - artifact push workflow target --destination target-ARM + - name: Build & Test s390x + dependencies: ["Validation"] + run: + when: "pull_request =~ '.*'" + task: + jobs: + - name: Build & Test s390x ubi9 + commands: + - export OS_TAG="-ubi9" + - ci-tools ci-update-version --direct-pom-edit + - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") + - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" + # Register QEMU binfmt for s390x so Docker can run s390x containers on amd64 for testing. + # ARM builds run on native arm64 machines, but s390x cross-compiles on amd64 via BuildX and needs QEMU. + - docker run --privileged --rm tonistiigi/binfmt --install s390x + - 'docker buildx ls | grep -q s390x || (echo "ERROR: s390x binfmt registration failed" && exit 1)' + - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker-fabric8 clean install dependency:analyze validate -U $S390X_MVN_PL_ARGS -Ddocker.registry=$DOCKER_DEV_REGISTRY + -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$S390X_ARCH + -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$S390X_ARCH -Ddocker.os_type=ubi9 -Ddocker.buildx.platforms=linux/s390x $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true + - . cache-maven store + - export S390X_DOCKER_DEV_FULL_IMAGES=$DOCKER_DEV_REGISTRY${S390X_DOCKER_REPOS// /:$DOCKER_DEV_TAG$OS_TAG $DOCKER_DEV_REGISTRY}:$DOCKER_DEV_TAG$OS_TAG + - export S390X_DOCKER_DEV_FULL_IMAGES=${S390X_DOCKER_DEV_FULL_IMAGES// /$S390X_ARCH }$S390X_ARCH + - for image in $S390X_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done + epilogue: + always: + commands: + - . publish-test-results + - artifact push workflow target/test-results after_pipeline: task: agent: