From f8873dfbebaaf47f3dcc76e4102020d6e282c4e9 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Wed, 8 Oct 2025 23:20:37 -0700 Subject: [PATCH 01/20] Add build with Docker image and s3 upload --- .semaphore/semaphore.yml | 35 ++++++++++++++++++++++++ Makefile | 54 +++++++++++++++++++++++++++++++++++++- docker/Dockerfile_muckrake | 29 ++++++++++++++++++++ 3 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile_muckrake diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 845c2def5b..9a7a5fee05 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -88,6 +88,41 @@ blocks: commands: - test-results publish . -N "windows/amd64" + - name: "Muckrake Check" + dependencies: [] + task: + jobs: + - name: "Build & Upload linux/amd64" + commands: + - checkout + - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" + - echo "Build ID: ${BUILD_ID}" + - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com + - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest + - GOOS=linux GOARCH=amd64 make muckrake-build-docker + - . assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore + - BUILD_ID="${BUILD_ID}" BINARY_PATH="dist/confluent_linux_amd64/confluent" S3_KEY="confluent_linux_amd64" make muckrake-upload-s3 + - echo "✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/${BUILD_ID}/" + + - name: "Muckrake Check linux/arm64" + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu24-04-arm64-1 + jobs: + - name: "Build & Upload linux/arm64" + commands: + - checkout + - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" + - echo "Build ID: ${BUILD_ID}" + - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com + - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest + - GOOS=linux GOARCH=arm64 make muckrake-build-docker + - . assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore + - BUILD_ID="${BUILD_ID}" BINARY_PATH="dist/confluent_linux_arm64/confluent" S3_KEY="confluent_linux_arm64" make muckrake-upload-s3 + - echo "✅ linux/arm64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/${BUILD_ID}/" + after_pipeline: task: jobs: diff --git a/Makefile b/Makefile index 75bbe67de9..7d29e76232 100644 --- a/Makefile +++ b/Makefile @@ -140,4 +140,56 @@ coverage: ## Merge coverage data from unit and integration tests into coverage.t @tail -n +2 coverage.unit.out >> coverage.txt @tail -n +2 coverage.integration.out >> coverage.txt @echo "Coverage data saved to: coverage.txt" - @artifact push workflow coverage.txt \ No newline at end of file + @artifact push workflow coverage.txt + +.PHONY: muckrake-build-docker +muckrake-build-docker: + @echo "Building CLI binary with Docker image for Muckrake Check..." + @echo "Target: $(GOOS)/$(GOARCH)" + @echo "Clean up any possible existing containers under same name..." + @docker rm -f confluent-muckrake-builder-container 2>/dev/null || true + @echo "Building Docker image..." + docker build -f docker/Dockerfile_muckrake \ + --build-arg GOOS=$(GOOS) \ + --build-arg GOARCH=$(GOARCH) \ + -t confluent-muckrake-builder . + @echo "Creating a container from the image..." + docker create --name confluent-muckrake-builder-container confluent-muckrake-builder + @echo "Copy built binary from the container" + docker cp confluent-muckrake-builder-container:/workspace/dist/. dist/ + @echo "Clean up container" + docker rm confluent-muckrake-builder-container + @echo "✅ Binary built successfully at dist/confluent_$(GOOS)_$(GOARCH)/confluent" + +.PHONY: muckrake-upload-s3 +muckrake-upload-s3: + @echo "Uploading CLI binary to S3..." + @if [ -z "$(BUILD_ID)" ]; then \ + echo "❌ Error: BUILD_ID is not set"; \ + exit 1; \ + fi + @if [ -z "$(BINARY_PATH)" ]; then \ + echo "❌ Error: BINARY_PATH is not set"; \ + exit 1; \ + fi + @if [ -z "$(S3_KEY)" ]; then \ + echo "❌ Error: S3_KEY is not set"; \ + exit 1; \ + fi + @if [ ! -f "$(BINARY_PATH)" ]; then \ + echo "❌ Error: Binary not found at $(BINARY_PATH)"; \ + exit 1; \ + fi + @echo "Build ID: $(BUILD_ID)" + @echo "Binary: $(BINARY_PATH)" + @echo "S3 Key: $(S3_KEY)" + @aws s3 cp $(BINARY_PATH) s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY) \ + --acl public-read --region us-west-2 + @echo "✅ Binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY)" + +.PHONY: muckrake-clean +muckrake-clean: + @echo "Cleaning up Muckrake Check artifacts..." + @docker rm -f confluent-muckrake-builder-container 2>/dev/null || true + @docker rmi -f confluent-muckrake-builder 2>/dev/null || true + @echo "✅ Cleanup complete" \ No newline at end of file diff --git a/docker/Dockerfile_muckrake b/docker/Dockerfile_muckrake new file mode 100644 index 0000000000..79dab38ac9 --- /dev/null +++ b/docker/Dockerfile_muckrake @@ -0,0 +1,29 @@ +# Uses cc-cli Docker image to build CLI binaries + +ARG GOOS=linux +ARG GOARCH=amd64 + +FROM 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest + +WORKDIR /workspace + +COPY . . + +ARG GOOS +ARG GOARCH + +RUN set -ex && \ + go mod download && \ + go mod verify && \ + mkdir -p dist/confluent_${GOOS}_${GOARCH} && \ + COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "dev") && \ + BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) && \ + echo "Building for ${GOOS}/${GOARCH} (commit: ${COMMIT_SHA}, date: ${BUILD_DATE})" && \ + CGO_ENABLED=1 GOOS=${GOOS} GOARCH=${GOARCH} go build \ + -ldflags="-s -w -X main.commit=${COMMIT_SHA} -X main.date=${BUILD_DATE}" \ + -o dist/confluent_${GOOS}_${GOARCH}/confluent \ + ./cmd/confluent && \ + ls -lh dist/confluent_${GOOS}_${GOARCH}/confluent + +CMD ["/bin/sh"] + From e7ddbccb5e4eb8fc7ac0ad334400d124ab84c4b0 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Wed, 8 Oct 2025 23:49:29 -0700 Subject: [PATCH 02/20] Update Dockerfile_muckrake --- docker/Dockerfile_muckrake | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile_muckrake b/docker/Dockerfile_muckrake index 79dab38ac9..c8b44e3029 100644 --- a/docker/Dockerfile_muckrake +++ b/docker/Dockerfile_muckrake @@ -5,25 +5,29 @@ ARG GOARCH=amd64 FROM 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest -WORKDIR /workspace - -COPY . . - ARG GOOS ARG GOARCH -RUN set -ex && \ +SHELL ["/bin/bash", "-c"] + +COPY . /workspace/ + +RUN cd /workspace/ && \ + set -ex && \ go mod download && \ go mod verify && \ mkdir -p dist/confluent_${GOOS}_${GOARCH} && \ COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "dev") && \ BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) && \ - echo "Building for ${GOOS}/${GOARCH} (commit: ${COMMIT_SHA}, date: ${BUILD_DATE})" && \ - CGO_ENABLED=1 GOOS=${GOOS} GOARCH=${GOARCH} go build \ + echo "Building Confluent Development CLI for ${GOOS}/${GOARCH}" && \ + echo " Commit: ${COMMIT_SHA}" && \ + echo " Date: ${BUILD_DATE}" && \ + CGO_ENABLED=1 GOEXPERIMENT=boringcrypto GOOS=${GOOS} GOARCH=${GOARCH} go build \ -ldflags="-s -w -X main.commit=${COMMIT_SHA} -X main.date=${BUILD_DATE}" \ -o dist/confluent_${GOOS}_${GOARCH}/confluent \ ./cmd/confluent && \ - ls -lh dist/confluent_${GOOS}_${GOARCH}/confluent + ls -lh dist/confluent_${GOOS}_${GOARCH}/confluent && \ + echo "✅ Build completed successfully" -CMD ["/bin/sh"] +WORKDIR /workspace From 4e2f38d2c55c092981119f1e2476a07bf8b4a294 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 11:11:43 -0700 Subject: [PATCH 03/20] Fix error --- .semaphore/semaphore.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9a7a5fee05..c2e8a38220 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -88,9 +88,12 @@ blocks: commands: - test-results publish . -N "windows/amd64" - - name: "Muckrake Check" + - name: "Muckrake Check linux/amd64" dependencies: [] task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 jobs: - name: "Build & Upload linux/amd64" commands: From 2edca7eea85c5cafb6692e9cc68485c34c2e681c Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 11:41:07 -0700 Subject: [PATCH 04/20] Fix error --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index c2e8a38220..da9cd2c495 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -98,7 +98,7 @@ blocks: - name: "Build & Upload linux/amd64" commands: - checkout - - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" + - 'export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}"' - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest @@ -117,7 +117,7 @@ blocks: - name: "Build & Upload linux/arm64" commands: - checkout - - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" + - 'export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}"' - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest From f0f6f6b348828439669a380a2616b51d49b34b1b Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 12:02:46 -0700 Subject: [PATCH 05/20] Fix error --- .semaphore/semaphore.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index da9cd2c495..9694a3ca11 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -98,12 +98,12 @@ blocks: - name: "Build & Upload linux/amd64" commands: - checkout - - 'export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}"' + - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest - GOOS=linux GOARCH=amd64 make muckrake-build-docker - - . assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore + - source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore - BUILD_ID="${BUILD_ID}" BINARY_PATH="dist/confluent_linux_amd64/confluent" S3_KEY="confluent_linux_amd64" make muckrake-upload-s3 - echo "✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/${BUILD_ID}/" @@ -117,12 +117,12 @@ blocks: - name: "Build & Upload linux/arm64" commands: - checkout - - 'export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}"' + - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest - GOOS=linux GOARCH=arm64 make muckrake-build-docker - - . assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore + - source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore - BUILD_ID="${BUILD_ID}" BINARY_PATH="dist/confluent_linux_arm64/confluent" S3_KEY="confluent_linux_arm64" make muckrake-upload-s3 - echo "✅ linux/arm64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/${BUILD_ID}/" From a30f6fffce4256ca5de7d82954312472df1575c3 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 12:32:13 -0700 Subject: [PATCH 06/20] Fix error --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9694a3ca11..e0b1d53a24 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -98,7 +98,7 @@ blocks: - name: "Build & Upload linux/amd64" commands: - checkout - - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" + - "export BUILD_ID=\"${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}\"" - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest @@ -117,7 +117,7 @@ blocks: - name: "Build & Upload linux/arm64" commands: - checkout - - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}" + - "export BUILD_ID=\"${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}\"" - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest From 50c0890daea542392937b3a9ff9095c83c55bd46 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 12:53:40 -0700 Subject: [PATCH 07/20] Fix error --- .semaphore/semaphore.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e0b1d53a24..7faebda612 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -98,7 +98,8 @@ blocks: - name: "Build & Upload linux/amd64" commands: - checkout - - "export BUILD_ID=\"${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}\"" + - export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8) + - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest @@ -117,7 +118,8 @@ blocks: - name: "Build & Upload linux/arm64" commands: - checkout - - "export BUILD_ID=\"${SEMAPHORE_GIT_BRANCH}-${SEMAPHORE_GIT_SHA:0:8}-${SEMAPHORE_WORKFLOW_ID}\"" + - export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8) + - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - echo "Build ID: ${BUILD_ID}" - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest From 573c98885de5ea3c206f3bc7e49ada816053cc04 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 13:17:55 -0700 Subject: [PATCH 08/20] Fix error --- .semaphore/semaphore.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 7faebda612..7c5564d05d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -98,15 +98,15 @@ blocks: - name: "Build & Upload linux/amd64" commands: - checkout - - export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8) - - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - - echo "Build ID: ${BUILD_ID}" - - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest - - GOOS=linux GOARCH=amd64 make muckrake-build-docker - - source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore - - BUILD_ID="${BUILD_ID}" BINARY_PATH="dist/confluent_linux_amd64/confluent" S3_KEY="confluent_linux_amd64" make muckrake-upload-s3 - - echo "✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/${BUILD_ID}/" + - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" + - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" + - "echo Build ID: $BUILD_ID" + - "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com" + - "docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" + - "GOOS=linux GOARCH=amd64 make muckrake-build-docker" + - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" + - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_amd64/confluent S3_KEY=confluent_linux_amd64 make muckrake-upload-s3" + - "echo ✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" - name: "Muckrake Check linux/arm64" dependencies: [] @@ -118,15 +118,15 @@ blocks: - name: "Build & Upload linux/arm64" commands: - checkout - - export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8) - - export BUILD_ID="${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - - echo "Build ID: ${BUILD_ID}" - - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com - - docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest - - GOOS=linux GOARCH=arm64 make muckrake-build-docker - - source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore - - BUILD_ID="${BUILD_ID}" BINARY_PATH="dist/confluent_linux_arm64/confluent" S3_KEY="confluent_linux_arm64" make muckrake-upload-s3 - - echo "✅ linux/arm64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/${BUILD_ID}/" + - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" + - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" + - "echo Build ID: $BUILD_ID" + - "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com" + - "docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" + - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" + - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" + - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_arm64/confluent S3_KEY=confluent_linux_arm64 make muckrake-upload-s3" + - "echo ✅ linux/arm64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" after_pipeline: task: From 53028a3c59cd88caee08f8e276f3a5f7b2fbabed Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 13:34:58 -0700 Subject: [PATCH 09/20] Fix Test linux/amd64 error --- debian/patches/standard_build_layout.patch | 292 ++++++++++++--------- 1 file changed, 162 insertions(+), 130 deletions(-) diff --git a/debian/patches/standard_build_layout.patch b/debian/patches/standard_build_layout.patch index d1c3592bcd..b50d3eaa6e 100644 --- a/debian/patches/standard_build_layout.patch +++ b/debian/patches/standard_build_layout.patch @@ -1,6 +1,6 @@ ---- cli/Makefile 2025-08-27 10:15:22.860561026 -0700 -+++ debian/Makefile 2025-09-12 11:15:01.286651273 -0700 -@@ -1,143 +1,163 @@ +--- cli/Makefile 2025-10-08 23:48:22 ++++ debian/Makefile 2025-09-23 10:17:30 +@@ -1,195 +1,163 @@ -SHELL := /bin/bash -GORELEASER_VERSION := v1.21.2 +SHELL=/bin/bash @@ -19,8 +19,10 @@ - endif -else # darwin - $(MAKE) cli-builder --endif -- ++ifndef VERSION ++ VERSION=$(CLI_VERSION) + endif + -# Cross-compile from darwin to any of the OS/Arch pairs below -.PHONY: cross-build -cross-build: @@ -30,7 +32,25 @@ - else # darwin/arm64 - $(MAKE) cli-builder - endif --else ++export PACKAGE_TITLE=cli ++export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) ++export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) ++ ++# Defaults that are likely to vary by platform. These are cleanly separated so ++# it should be easy to maintain altered values on platform-specific branches ++# when the values aren't overridden by the script invoking the Makefile ++ ++APPLY_PATCHES?=yes ++ ++# DESTDIR may be overridden by e.g. debian packaging ++ifeq ($(DESTDIR),) ++DESTDIR=$(CURDIR)/BUILD/ ++endif ++ ++ifeq ($(PACKAGE_TYPE),archive) ++PREFIX=$(PACKAGE_NAME) ++SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) + else - ifeq ($(GOOS),windows) # windows/amd64 - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CGO_LDFLAGS="-fstack-protector -static" $(MAKE) cli-builder - else ifeq ($(GOOS),linux) # linux/amd64 @@ -38,12 +58,17 @@ - else # darwin/amd64 - $(MAKE) cli-builder - endif --endif -- ++PREFIX=/usr ++SYSCONFDIR=/etc/$(PACKAGE_TITLE) + endif + -.PHONY: cli-builder -cli-builder: - GOOS="" GOARCH="" CC="" CXX="" CGO_LDFLAGS="" go install github.com/goreleaser/goreleaser@$(GORELEASER_VERSION) -- ++_empty := ++_space := $(_empty) $(empty) ++split_version := $(subst .,$(_space),$(CLI_VERSION)) + -ifeq ($(GOLANG_FIPS),1) - wget "https://go.dev/dl/go$$(cat .go-version).src.tar.gz" && \ - tar -xf go$$(cat .go-version).src.tar.gz && \ @@ -60,37 +85,6 @@ - rm -rf go go-openssl go$$(cat .go-version).src.tar.gz -else - TAGS=$(TAGS) CC=$(CC) CXX=$(CXX) CGO_LDFLAGS=$(CGO_LDFLAGS) goreleaser build --clean --single-target --snapshot -+ifndef VERSION -+ VERSION=$(CLI_VERSION) - endif - -+export PACKAGE_TITLE=cli -+export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) -+export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) -+ -+# Defaults that are likely to vary by platform. These are cleanly separated so -+# it should be easy to maintain altered values on platform-specific branches -+# when the values aren't overridden by the script invoking the Makefile -+ -+APPLY_PATCHES?=yes -+ -+# DESTDIR may be overridden by e.g. debian packaging -+ifeq ($(DESTDIR),) -+DESTDIR=$(CURDIR)/BUILD/ -+endif -+ -+ifeq ($(PACKAGE_TYPE),archive) -+PREFIX=$(PACKAGE_NAME) -+SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) -+else -+PREFIX=/usr -+SYSCONFDIR=/etc/$(PACKAGE_TITLE) -+endif -+ -+_empty := -+_space := $(_empty) $(empty) -+split_version := $(subst .,$(_space),$(CLI_VERSION)) -+ +all: install + +archive: install @@ -101,12 +95,12 @@ +ifeq ($(APPLY_PATCHES),yes) + git reset --hard HEAD + cat debian/patches/series | xargs -iPATCH bash -c 'patch -p1 < debian/patches/PATCH' -+endif -+ + endif + +BINPATH=$(PREFIX)/bin +LIBPATH=$(PREFIX)/libexec/$(PACKAGE_TITLE) +DOCPATH=$(PREFIX)/share/doc/$(PACKAGE_TITLE) -+ + +# Notes on the archive download links: +# As of v3.0.0, the version number no longer has the 'v' prefix +# As of v3.43.0, we download the '_disableupdates' variant @@ -114,15 +108,27 @@ +install: apply-patches + rm -rf $(DESTDIR)$(PREFIX) + mkdir -p $(DESTDIR)$(PREFIX) -+ + +-.PHONY: clean +-clean: +- for dir in bin dist docs legal prebuilt release-notes; do \ +- [ -d $$dir ] && rm -r $$dir || true; \ +- done + mkdir -p $(DESTDIR)$(BINPATH) + mkdir -p $(DESTDIR)$(LIBPATH) + mkdir -p $(DESTDIR)$(DOCPATH) + mkdir -p $(DESTDIR)$(SYSCONFDIR) -+ + +-.PHONY: lint +-lint: lint-go lint-cli + cp packaging/confluent.sh $(DESTDIR)$(BINPATH)/confluent + chmod 755 $(DESTDIR)$(BINPATH)/confluent -+ + +-.PHONY: lint-go +-lint-go: +- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ +- golangci-lint run --timeout 10m +- @echo "✅ golangci-lint" + cd $(DESTDIR)$(LIBPATH); \ + v=""; if [[ $(word 1,$(split_version)) -eq 2 ]]; then v="v"; fi; \ + disableupdates=""; if [[ $(word 1,$(split_version)) -ge 3 && $(word 2,$(split_version)) -ge 43 ]]; then disableupdates="_disableupdates"; fi; \ @@ -142,38 +148,25 @@ + filepath=windows_amd64/confluent.exe; \ + curl -fs https://$${baseurl}/confluent-cli/binaries/$(CLI_VERSION)/confluent$${version}_windows_amd64.exe -o $${filepath}; \ + chmod 755 $${filepath} -+ -+ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE -+ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt -+ chown -R root:root $(DESTDIR)$(PREFIX) - --.PHONY: clean - clean: -- for dir in bin dist docs legal prebuilt release-notes; do \ -- [ -d $$dir ] && rm -r $$dir || true; \ -- done -- --.PHONY: lint --lint: lint-go lint-cli -- --.PHONY: lint-go --lint-go: -- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ -- golangci-lint run --timeout 10m -- @echo "✅ golangci-lint" -- -.PHONY: lint-cli -lint-cli: cmd/lint/en_US.aff cmd/lint/en_US.dic - go run cmd/lint/main.go -aff-file $(word 1,$^) -dic-file $(word 2,$^) $(ARGS) - @echo "✅ cmd/lint/main.go" -- ++ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE ++ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt + -cmd/lint/en_US.aff: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.aff?format=TEXT" | base64 -D > $@ -- ++ chown -R root:root $(DESTDIR)$(PREFIX) + -cmd/lint/en_US.dic: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.dic?format=TEXT" | base64 -D > $@ -- ++clean: ++ rm -rf $(CURDIR)/$(PACKAGE_NAME)* ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm ++ rm -rf RPM_BUILDING + -.PHONY: unit-test -unit-test: -ifdef CI @@ -181,19 +174,25 @@ - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v -race -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) -else - go test -timeout 0 -v -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) $(UNIT_TEST_ARGS) --endif -+ rm -rf $(CURDIR)/$(PACKAGE_NAME)* -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm -+ rm -rf RPM_BUILDING -+ +distclean: clean +ifneq ($(PACKAGE_TYPE),deb) + git reset --hard HEAD + git status --ignored --porcelain | cut -d ' ' -f 2 | xargs rm -rf -+endif -+ + endif + +-.PHONY: build-for-integration-test +-build-for-integration-test: +-ifdef CI +- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent ./cmd/confluent +-else +- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent ./cmd/confluent +-endif +RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//' -e 's/-cp[0-9]*//' -e 's/-hotfix[0-9]*//' -e 's/-[0-9]*//') -+ + +-.PHONY: build-for-integration-test-windows +-build-for-integration-test-windows: +-ifdef CI +- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent +# Get any -alpha, -beta (preview), -rc (release candidate), -SNAPSHOT (nightly), -cp (confluent patch), -hotfix piece that we need to put into the Release part of +# the version since RPM versions don't support non-numeric +# characters. Ultimately, for something like 0.8.2-beta, we want to end up with @@ -205,16 +204,36 @@ +ifneq ($(RPM_RELEASE_POSTFIX),) + RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) + RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) -+else + else +- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent + RPM_RELEASE_ID=$(REVISION) -+endif -+ + endif + +-.PHONY: integration-test +-integration-test: +-ifdef CI +- go install gotest.tools/gotestsum@v1.12.1 && \ +- export GOCOVERDIR=test/coverage && \ +- rm -rf $${GOCOVERDIR} && mkdir $${GOCOVERDIR} && \ +- gotestsum --junitfile integration-test-report.xml -- -timeout 0 -v -race $$(go list ./... | grep github.com/confluentinc/cli/v4/test) && \ +- go tool covdata textfmt -i $${GOCOVERDIR} -o coverage.integration.out +-else +- export GOCOVERDIR=test/coverage && \ +- rm -rf $${GOCOVERDIR} && mkdir $${GOCOVERDIR} && \ +- go test -timeout 0 -v $$(go list ./... | grep github.com/confluentinc/cli/v4/test) $(INTEGRATION_TEST_ARGS) && \ +- go tool covdata textfmt -i $${GOCOVERDIR} -o coverage.integration.out +-endif +rpm: rpm-amd64-build rpm-arm64-build -+ + +-.PHONY: test +-test: unit-test integration-test +rpm-amd64-build: rpm-amd64-spec + echo "Building the amd64 RPM" + rpmbuild --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz -+ + +-.PHONY: generate-packaging-patch +-generate-packaging-patch: +- diff -u Makefile debian/Makefile | sed "1 s_Makefile_cli/Makefile_" > debian/patches/standard_build_layout.patch +rpm-arm64-build: rpm-arm64-spec + echo "Building the arm64 RPM" + rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz @@ -224,7 +243,15 @@ + echo "The RPMs have been created and can be found here:" + @ls -laF $(FULL_PACKAGE_TITLE)*rpm + echo "=================================================" -+ + +-.PHONY: coverage +-coverage: ## Merge coverage data from unit and integration tests into coverage.txt +- @echo "Merging coverage data..." +- @echo "mode: atomic" > coverage.txt +- @tail -n +2 coverage.unit.out >> coverage.txt +- @tail -n +2 coverage.integration.out >> coverage.txt +- @echo "Coverage data saved to: coverage.txt" +- @artifact push workflow coverage.txt +# Unfortunately, because of version naming issues and the way rpmbuild expects +# the paths in the tar file to be named, we need to rearchive the package. So +# instead of depending on archive, this target just uses the unarchived, @@ -238,7 +265,25 @@ + sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/arm64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec + rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ + +-.PHONY: muckrake-build-docker +-muckrake-build-docker: +- @echo "Building CLI binary with Docker image for Muckrake Check..." +- @echo "Target: $(GOOS)/$(GOARCH)" +- @echo "Clean up any possible existing containers under same name..." +- @docker rm -f confluent-muckrake-builder-container 2>/dev/null || true +- @echo "Building Docker image..." +- docker build -f docker/Dockerfile_muckrake \ +- --build-arg GOOS=$(GOOS) \ +- --build-arg GOARCH=$(GOARCH) \ +- -t confluent-muckrake-builder . +- @echo "Creating a container from the image..." +- docker create --name confluent-muckrake-builder-container confluent-muckrake-builder +- @echo "Copy built binary from the container" +- docker cp confluent-muckrake-builder-container:/workspace/dist/. dist/ +- @echo "Clean up container" +- docker rm confluent-muckrake-builder-container +- @echo "✅ Binary built successfully at dist/confluent_$(GOOS)_$(GOARCH)/confluent" +rpm-arm64-spec: + rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) @@ -247,55 +292,42 @@ + rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) --.PHONY: build-for-integration-test --build-for-integration-test: --ifdef CI -- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent ./cmd/confluent --else -- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent ./cmd/confluent --endif +-.PHONY: muckrake-upload-s3 +-muckrake-upload-s3: +- @echo "Uploading CLI binary to S3..." +- @if [ -z "$(BUILD_ID)" ]; then \ +- echo "❌ Error: BUILD_ID is not set"; \ +- exit 1; \ +- fi +- @if [ -z "$(BINARY_PATH)" ]; then \ +- echo "❌ Error: BINARY_PATH is not set"; \ +- exit 1; \ +- fi +- @if [ -z "$(S3_KEY)" ]; then \ +- echo "❌ Error: S3_KEY is not set"; \ +- exit 1; \ +- fi +- @if [ ! -f "$(BINARY_PATH)" ]; then \ +- echo "❌ Error: Binary not found at $(BINARY_PATH)"; \ +- exit 1; \ +- fi +- @echo "Build ID: $(BUILD_ID)" +- @echo "Binary: $(BINARY_PATH)" +- @echo "S3 Key: $(S3_KEY)" +- @aws s3 cp $(BINARY_PATH) s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY) \ +- --acl public-read --region us-west-2 +- @echo "✅ Binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY)" +rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS --.PHONY: build-for-integration-test-windows --build-for-integration-test-windows: --ifdef CI -- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent --else -- go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent --endif +-.PHONY: muckrake-clean +-muckrake-clean: +- @echo "Cleaning up Muckrake Check artifacts..." +- @docker rm -f confluent-muckrake-builder-container 2>/dev/null || true +- @docker rmi -f confluent-muckrake-builder 2>/dev/null || true +- @echo "✅ Cleanup complete" +\ No newline at end of file +RPM_BUILDING/%: + mkdir -p $@ - --.PHONY: integration-test --integration-test: --ifdef CI -- go install gotest.tools/gotestsum@v1.12.1 && \ -- export GOCOVERDIR=test/coverage && \ -- rm -rf $${GOCOVERDIR} && mkdir $${GOCOVERDIR} && \ -- gotestsum --junitfile integration-test-report.xml -- -timeout 0 -v -race $$(go list ./... | grep github.com/confluentinc/cli/v4/test) && \ -- go tool covdata textfmt -i $${GOCOVERDIR} -o coverage.integration.out --else -- export GOCOVERDIR=test/coverage && \ -- rm -rf $${GOCOVERDIR} && mkdir $${GOCOVERDIR} && \ -- go test -timeout 0 -v $$(go list ./... | grep github.com/confluentinc/cli/v4/test) $(INTEGRATION_TEST_ARGS) && \ -- go tool covdata textfmt -i $${GOCOVERDIR} -o coverage.integration.out --endif -- --.PHONY: test --test: unit-test integration-test -- --.PHONY: generate-packaging-patch --generate-packaging-patch: -- diff -u Makefile debian/Makefile | sed "1 s_Makefile_cli/Makefile_" > debian/patches/standard_build_layout.patch -- --.PHONY: coverage --coverage: ## Merge coverage data from unit and integration tests into coverage.txt -- @echo "Merging coverage data..." -- @echo "mode: atomic" > coverage.txt -- @tail -n +2 coverage.unit.out >> coverage.txt -- @tail -n +2 coverage.integration.out >> coverage.txt -- @echo "Coverage data saved to: coverage.txt" -- @artifact push workflow coverage.txt -\ No newline at end of file ++ +RELEASE_%: + echo 0 > $@ From b528cc0a465bf5b3cbe784c43de978766533b8c0 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 14:09:57 -0700 Subject: [PATCH 10/20] Fix Muckrake Check linux/amd64 error --- .semaphore/semaphore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 7c5564d05d..0065e427bf 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -101,6 +101,7 @@ blocks: - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - "echo Build ID: $BUILD_ID" + - 'mkdir -p ~/.docker && echo "{}" > ~/.docker/config.json' - "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com" - "docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" - "GOOS=linux GOARCH=amd64 make muckrake-build-docker" @@ -121,6 +122,7 @@ blocks: - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - "echo Build ID: $BUILD_ID" + - 'mkdir -p ~/.docker && echo "{}" > ~/.docker/config.json' - "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com" - "docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" From 28dc79a9b5f8c116c857a862266956fc05542ec8 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 14:28:28 -0700 Subject: [PATCH 11/20] Fix Muckrake Check linux/amd64 error --- .semaphore/semaphore.yml | 8 ++++---- docker/Dockerfile_muckrake | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 0065e427bf..2b917765cd 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -102,8 +102,8 @@ blocks: - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - "echo Build ID: $BUILD_ID" - 'mkdir -p ~/.docker && echo "{}" > ~/.docker/config.json' - - "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com" - - "docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" + - "aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-west-2.amazonaws.com" + - "docker pull 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" - "GOOS=linux GOARCH=amd64 make muckrake-build-docker" - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_amd64/confluent S3_KEY=confluent_linux_amd64 make muckrake-upload-s3" @@ -123,8 +123,8 @@ blocks: - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - "echo Build ID: $BUILD_ID" - 'mkdir -p ~/.docker && echo "{}" > ~/.docker/config.json' - - "aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-east-1.amazonaws.com" - - "docker pull 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" + - "aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-west-2.amazonaws.com" + - "docker pull 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_arm64/confluent S3_KEY=confluent_linux_arm64 make muckrake-upload-s3" diff --git a/docker/Dockerfile_muckrake b/docker/Dockerfile_muckrake index c8b44e3029..e80b3edead 100644 --- a/docker/Dockerfile_muckrake +++ b/docker/Dockerfile_muckrake @@ -3,7 +3,7 @@ ARG GOOS=linux ARG GOARCH=amd64 -FROM 519856050701.dkr.ecr.us-east-1.amazonaws.com/docker/dev/confluentinc/cc-cli:latest +FROM 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cc-cli:latest ARG GOOS ARG GOARCH From 71a07a9c4ae80d0e462f8f8535219158f10f73ad Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 14:45:21 -0700 Subject: [PATCH 12/20] Remove docker ecr blob --- .semaphore/semaphore.yml | 6 ------ docker/Dockerfile_muckrake | 23 ++++++++++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 2b917765cd..68f61c8431 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -101,9 +101,6 @@ blocks: - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - "echo Build ID: $BUILD_ID" - - 'mkdir -p ~/.docker && echo "{}" > ~/.docker/config.json' - - "aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-west-2.amazonaws.com" - - "docker pull 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" - "GOOS=linux GOARCH=amd64 make muckrake-build-docker" - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_amd64/confluent S3_KEY=confluent_linux_amd64 make muckrake-upload-s3" @@ -122,9 +119,6 @@ blocks: - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" - "echo Build ID: $BUILD_ID" - - 'mkdir -p ~/.docker && echo "{}" > ~/.docker/config.json' - - "aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 519856050701.dkr.ecr.us-west-2.amazonaws.com" - - "docker pull 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cc-cli:latest" - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_arm64/confluent S3_KEY=confluent_linux_arm64 make muckrake-upload-s3" diff --git a/docker/Dockerfile_muckrake b/docker/Dockerfile_muckrake index e80b3edead..4049aa55c1 100644 --- a/docker/Dockerfile_muckrake +++ b/docker/Dockerfile_muckrake @@ -1,19 +1,36 @@ -# Uses cc-cli Docker image to build CLI binaries +# AlmaLinux 8.10 chosen for current GLIBC requirement of 2.28 +# This matches the base image used in the existing build Dockerfiles ARG GOOS=linux ARG GOARCH=amd64 -FROM 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/confluentinc/cc-cli:latest +FROM almalinux:8.10 ARG GOOS ARG GOARCH SHELL ["/bin/bash", "-c"] +# Install required packages +RUN yum -y update && \ + yum -y install make git wget gcc-toolset-9 && \ + scl enable gcc-toolset-9 bash + +# Copy source code COPY . /workspace/ +# Install Go and build the binary RUN cd /workspace/ && \ set -ex && \ + export GO_VERSION=$(cat .go-version) && \ + if [ "${GOARCH}" = "amd64" ]; then \ + wget "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" && \ + tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"; \ + elif [ "${GOARCH}" = "arm64" ]; then \ + wget "https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz" && \ + tar -C /usr/local -xzf "go${GO_VERSION}.linux-arm64.tar.gz"; \ + fi && \ + export PATH=${PATH}:/usr/local/go/bin:/root/go/bin && \ go mod download && \ go mod verify && \ mkdir -p dist/confluent_${GOOS}_${GOARCH} && \ @@ -22,7 +39,7 @@ RUN cd /workspace/ && \ echo "Building Confluent Development CLI for ${GOOS}/${GOARCH}" && \ echo " Commit: ${COMMIT_SHA}" && \ echo " Date: ${BUILD_DATE}" && \ - CGO_ENABLED=1 GOEXPERIMENT=boringcrypto GOOS=${GOOS} GOARCH=${GOARCH} go build \ + CGO_ENABLED=1 GOEXPERIMENT=boringcrypto go build \ -ldflags="-s -w -X main.commit=${COMMIT_SHA} -X main.date=${BUILD_DATE}" \ -o dist/confluent_${GOOS}_${GOARCH}/confluent \ ./cmd/confluent && \ From ca8cc5b636c4d7c0927e8084568bc2f11010f586 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 15:03:57 -0700 Subject: [PATCH 13/20] Nit --- docker/Dockerfile_muckrake | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/Dockerfile_muckrake b/docker/Dockerfile_muckrake index 4049aa55c1..62d7efa58e 100644 --- a/docker/Dockerfile_muckrake +++ b/docker/Dockerfile_muckrake @@ -1,6 +1,3 @@ -# AlmaLinux 8.10 chosen for current GLIBC requirement of 2.28 -# This matches the base image used in the existing build Dockerfiles - ARG GOOS=linux ARG GOARCH=amd64 From 35b947b51323ca6f964e1cc3490451680d5fd20d Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 15:41:46 -0700 Subject: [PATCH 14/20] Nit --- docker/Dockerfile_muckrake | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/Dockerfile_muckrake b/docker/Dockerfile_muckrake index 62d7efa58e..7ef84d4880 100644 --- a/docker/Dockerfile_muckrake +++ b/docker/Dockerfile_muckrake @@ -8,15 +8,12 @@ ARG GOARCH SHELL ["/bin/bash", "-c"] -# Install required packages RUN yum -y update && \ yum -y install make git wget gcc-toolset-9 && \ scl enable gcc-toolset-9 bash -# Copy source code COPY . /workspace/ -# Install Go and build the binary RUN cd /workspace/ && \ set -ex && \ export GO_VERSION=$(cat .go-version) && \ From d1124cae2a707b055b16e7d77ace3d570297010d Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Thu, 9 Oct 2025 15:50:46 -0700 Subject: [PATCH 15/20] Improve BUILD_ID --- .semaphore/semaphore.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 68f61c8431..bff0302c09 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -99,7 +99,8 @@ blocks: commands: - checkout - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" + - "export TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" + - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}/${TIMESTAMP}-${SHORT_SHA}" - "echo Build ID: $BUILD_ID" - "GOOS=linux GOARCH=amd64 make muckrake-build-docker" - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" @@ -117,7 +118,8 @@ blocks: commands: - checkout - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}-${SHORT_SHA}-${SEMAPHORE_WORKFLOW_ID}" + - "export TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" + - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}/${TIMESTAMP}-${SHORT_SHA}" - "echo Build ID: $BUILD_ID" - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" From 31afec2243ab905b3f0f89f2107f65d628ab8d2f Mon Sep 17 00:00:00 2001 From: Steven Gagniere Date: Thu, 9 Oct 2025 16:30:31 -0700 Subject: [PATCH 16/20] generate packaging patch --- debian/patches/standard_build_layout.patch | 260 +++++++++++---------- 1 file changed, 140 insertions(+), 120 deletions(-) diff --git a/debian/patches/standard_build_layout.patch b/debian/patches/standard_build_layout.patch index b50d3eaa6e..f3497f43c4 100644 --- a/debian/patches/standard_build_layout.patch +++ b/debian/patches/standard_build_layout.patch @@ -1,5 +1,5 @@ ---- cli/Makefile 2025-10-08 23:48:22 -+++ debian/Makefile 2025-09-23 10:17:30 +--- cli/Makefile 2025-10-09 16:20:10.591464253 -0700 ++++ debian/Makefile 2025-10-08 10:35:28.087163073 -0700 @@ -1,195 +1,163 @@ -SHELL := /bin/bash -GORELEASER_VERSION := v1.21.2 @@ -19,10 +19,8 @@ - endif -else # darwin - $(MAKE) cli-builder -+ifndef VERSION -+ VERSION=$(CLI_VERSION) - endif - +-endif +- -# Cross-compile from darwin to any of the OS/Arch pairs below -.PHONY: cross-build -cross-build: @@ -32,25 +30,7 @@ - else # darwin/arm64 - $(MAKE) cli-builder - endif -+export PACKAGE_TITLE=cli -+export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) -+export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) -+ -+# Defaults that are likely to vary by platform. These are cleanly separated so -+# it should be easy to maintain altered values on platform-specific branches -+# when the values aren't overridden by the script invoking the Makefile -+ -+APPLY_PATCHES?=yes -+ -+# DESTDIR may be overridden by e.g. debian packaging -+ifeq ($(DESTDIR),) -+DESTDIR=$(CURDIR)/BUILD/ -+endif -+ -+ifeq ($(PACKAGE_TYPE),archive) -+PREFIX=$(PACKAGE_NAME) -+SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) - else +-else - ifeq ($(GOOS),windows) # windows/amd64 - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CGO_LDFLAGS="-fstack-protector -static" $(MAKE) cli-builder - else ifeq ($(GOOS),linux) # linux/amd64 @@ -58,17 +38,12 @@ - else # darwin/amd64 - $(MAKE) cli-builder - endif -+PREFIX=/usr -+SYSCONFDIR=/etc/$(PACKAGE_TITLE) - endif - +-endif +- -.PHONY: cli-builder -cli-builder: - GOOS="" GOARCH="" CC="" CXX="" CGO_LDFLAGS="" go install github.com/goreleaser/goreleaser@$(GORELEASER_VERSION) -+_empty := -+_space := $(_empty) $(empty) -+split_version := $(subst .,$(_space),$(CLI_VERSION)) - +- -ifeq ($(GOLANG_FIPS),1) - wget "https://go.dev/dl/go$$(cat .go-version).src.tar.gz" && \ - tar -xf go$$(cat .go-version).src.tar.gz && \ @@ -85,6 +60,37 @@ - rm -rf go go-openssl go$$(cat .go-version).src.tar.gz -else - TAGS=$(TAGS) CC=$(CC) CXX=$(CXX) CGO_LDFLAGS=$(CGO_LDFLAGS) goreleaser build --clean --single-target --snapshot ++ifndef VERSION ++ VERSION=$(CLI_VERSION) + endif + ++export PACKAGE_TITLE=cli ++export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) ++export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) ++ ++# Defaults that are likely to vary by platform. These are cleanly separated so ++# it should be easy to maintain altered values on platform-specific branches ++# when the values aren't overridden by the script invoking the Makefile ++ ++APPLY_PATCHES?=yes ++ ++# DESTDIR may be overridden by e.g. debian packaging ++ifeq ($(DESTDIR),) ++DESTDIR=$(CURDIR)/BUILD/ ++endif ++ ++ifeq ($(PACKAGE_TYPE),archive) ++PREFIX=$(PACKAGE_NAME) ++SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) ++else ++PREFIX=/usr ++SYSCONFDIR=/etc/$(PACKAGE_TITLE) ++endif ++ ++_empty := ++_space := $(_empty) $(empty) ++split_version := $(subst .,$(_space),$(CLI_VERSION)) ++ +all: install + +archive: install @@ -95,12 +101,12 @@ +ifeq ($(APPLY_PATCHES),yes) + git reset --hard HEAD + cat debian/patches/series | xargs -iPATCH bash -c 'patch -p1 < debian/patches/PATCH' - endif - ++endif ++ +BINPATH=$(PREFIX)/bin +LIBPATH=$(PREFIX)/libexec/$(PACKAGE_TITLE) +DOCPATH=$(PREFIX)/share/doc/$(PACKAGE_TITLE) - ++ +# Notes on the archive download links: +# As of v3.0.0, the version number no longer has the 'v' prefix +# As of v3.43.0, we download the '_disableupdates' variant @@ -108,27 +114,15 @@ +install: apply-patches + rm -rf $(DESTDIR)$(PREFIX) + mkdir -p $(DESTDIR)$(PREFIX) - --.PHONY: clean --clean: -- for dir in bin dist docs legal prebuilt release-notes; do \ -- [ -d $$dir ] && rm -r $$dir || true; \ -- done ++ + mkdir -p $(DESTDIR)$(BINPATH) + mkdir -p $(DESTDIR)$(LIBPATH) + mkdir -p $(DESTDIR)$(DOCPATH) + mkdir -p $(DESTDIR)$(SYSCONFDIR) - --.PHONY: lint --lint: lint-go lint-cli ++ + cp packaging/confluent.sh $(DESTDIR)$(BINPATH)/confluent + chmod 755 $(DESTDIR)$(BINPATH)/confluent - --.PHONY: lint-go --lint-go: -- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ -- golangci-lint run --timeout 10m -- @echo "✅ golangci-lint" ++ + cd $(DESTDIR)$(LIBPATH); \ + v=""; if [[ $(word 1,$(split_version)) -eq 2 ]]; then v="v"; fi; \ + disableupdates=""; if [[ $(word 1,$(split_version)) -ge 3 && $(word 2,$(split_version)) -ge 43 ]]; then disableupdates="_disableupdates"; fi; \ @@ -148,25 +142,38 @@ + filepath=windows_amd64/confluent.exe; \ + curl -fs https://$${baseurl}/confluent-cli/binaries/$(CLI_VERSION)/confluent$${version}_windows_amd64.exe -o $${filepath}; \ + chmod 755 $${filepath} ++ ++ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE ++ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt + ++ chown -R root:root $(DESTDIR)$(PREFIX) +-.PHONY: clean + clean: +- for dir in bin dist docs legal prebuilt release-notes; do \ +- [ -d $$dir ] && rm -r $$dir || true; \ +- done +- +-.PHONY: lint +-lint: lint-go lint-cli +- +-.PHONY: lint-go +-lint-go: +- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ +- golangci-lint run --timeout 10m +- @echo "✅ golangci-lint" +- -.PHONY: lint-cli -lint-cli: cmd/lint/en_US.aff cmd/lint/en_US.dic - go run cmd/lint/main.go -aff-file $(word 1,$^) -dic-file $(word 2,$^) $(ARGS) - @echo "✅ cmd/lint/main.go" -+ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE -+ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt - +- -cmd/lint/en_US.aff: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.aff?format=TEXT" | base64 -D > $@ -+ chown -R root:root $(DESTDIR)$(PREFIX) - +- -cmd/lint/en_US.dic: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.dic?format=TEXT" | base64 -D > $@ -+clean: -+ rm -rf $(CURDIR)/$(PACKAGE_NAME)* -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm -+ rm -rf RPM_BUILDING - +- -.PHONY: unit-test -unit-test: -ifdef CI @@ -174,11 +181,71 @@ - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v -race -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) -else - go test -timeout 0 -v -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) $(UNIT_TEST_ARGS) +-endif ++ rm -rf $(CURDIR)/$(PACKAGE_NAME)* ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm ++ rm -rf RPM_BUILDING ++ +distclean: clean +ifneq ($(PACKAGE_TYPE),deb) + git reset --hard HEAD + git status --ignored --porcelain | cut -d ' ' -f 2 | xargs rm -rf - endif ++endif ++ ++RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//' -e 's/-cp[0-9]*//' -e 's/-hotfix[0-9]*//' -e 's/-[0-9]*//') ++ ++# Get any -alpha, -beta (preview), -rc (release candidate), -SNAPSHOT (nightly), -cp (confluent patch), -hotfix piece that we need to put into the Release part of ++# the version since RPM versions don't support non-numeric ++# characters. Ultimately, for something like 0.8.2-beta, we want to end up with ++# Version=0.8.2 Release=0.X.beta ++# where X is the RPM release # of 0.8.2-beta (the prefix 0. forces this to be ++# considered earlier than any 0.8.2 final releases since those will start with ++# Version=0.8.2 Release=1) ++RPM_RELEASE_POSTFIX=$(subst -,,$(subst $(RPM_VERSION),,$(VERSION))) ++ifneq ($(RPM_RELEASE_POSTFIX),) ++ RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) ++ RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) ++else ++ RPM_RELEASE_ID=$(REVISION) ++endif ++ ++rpm: rpm-amd64-build rpm-arm64-build ++ ++rpm-amd64-build: rpm-amd64-spec ++ echo "Building the amd64 RPM" ++ rpmbuild --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz ++ ++rpm-arm64-build: rpm-arm64-spec ++ echo "Building the arm64 RPM" ++ rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz ++ find RPM_BUILDING/{,S}RPMS/ -type f | xargs -n1 -iXXX mv XXX . ++ echo ++ echo "=================================================" ++ echo "The RPMs have been created and can be found here:" ++ @ls -laF $(FULL_PACKAGE_TITLE)*rpm ++ echo "=================================================" ++ ++# Unfortunately, because of version naming issues and the way rpmbuild expects ++# the paths in the tar file to be named, we need to rearchive the package. So ++# instead of depending on archive, this target just uses the unarchived, ++# installed version to generate a new archive. Note that we always regenerate ++# the symlink because the RPM_VERSION doesn't include all the version info -- it ++# can leave of things like -beta, -rc1, etc. ++rpm-amd64-spec: rpm-build-area install debian/$(FULL_PACKAGE_TITLE).spec.in RELEASE_$(RPM_VERSION)$(RPM_RELEASE_POSTFIX_UNDERSCORE) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/arm64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec ++ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ ++rpm-arm64-spec: ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/amd64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec ++ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -.PHONY: build-for-integration-test -build-for-integration-test: @@ -187,27 +254,17 @@ -else - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent ./cmd/confluent -endif -+RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//' -e 's/-cp[0-9]*//' -e 's/-hotfix[0-9]*//' -e 's/-[0-9]*//') ++rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS -.PHONY: build-for-integration-test-windows -build-for-integration-test-windows: -ifdef CI - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent -+# Get any -alpha, -beta (preview), -rc (release candidate), -SNAPSHOT (nightly), -cp (confluent patch), -hotfix piece that we need to put into the Release part of -+# the version since RPM versions don't support non-numeric -+# characters. Ultimately, for something like 0.8.2-beta, we want to end up with -+# Version=0.8.2 Release=0.X.beta -+# where X is the RPM release # of 0.8.2-beta (the prefix 0. forces this to be -+# considered earlier than any 0.8.2 final releases since those will start with -+# Version=0.8.2 Release=1) -+RPM_RELEASE_POSTFIX=$(subst -,,$(subst $(RPM_VERSION),,$(VERSION))) -+ifneq ($(RPM_RELEASE_POSTFIX),) -+ RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) -+ RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) - else +-else - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent -+ RPM_RELEASE_ID=$(REVISION) - endif +-endif ++RPM_BUILDING/%: ++ mkdir -p $@ -.PHONY: integration-test -integration-test: @@ -223,27 +280,14 @@ - go test -timeout 0 -v $$(go list ./... | grep github.com/confluentinc/cli/v4/test) $(INTEGRATION_TEST_ARGS) && \ - go tool covdata textfmt -i $${GOCOVERDIR} -o coverage.integration.out -endif -+rpm: rpm-amd64-build rpm-arm64-build - +- -.PHONY: test -test: unit-test integration-test -+rpm-amd64-build: rpm-amd64-spec -+ echo "Building the amd64 RPM" -+ rpmbuild --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz - +- -.PHONY: generate-packaging-patch -generate-packaging-patch: - diff -u Makefile debian/Makefile | sed "1 s_Makefile_cli/Makefile_" > debian/patches/standard_build_layout.patch -+rpm-arm64-build: rpm-arm64-spec -+ echo "Building the arm64 RPM" -+ rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz -+ find RPM_BUILDING/{,S}RPMS/ -type f | xargs -n1 -iXXX mv XXX . -+ echo -+ echo "=================================================" -+ echo "The RPMs have been created and can be found here:" -+ @ls -laF $(FULL_PACKAGE_TITLE)*rpm -+ echo "=================================================" - +- -.PHONY: coverage -coverage: ## Merge coverage data from unit and integration tests into coverage.txt - @echo "Merging coverage data..." @@ -252,20 +296,7 @@ - @tail -n +2 coverage.integration.out >> coverage.txt - @echo "Coverage data saved to: coverage.txt" - @artifact push workflow coverage.txt -+# Unfortunately, because of version naming issues and the way rpmbuild expects -+# the paths in the tar file to be named, we need to rearchive the package. So -+# instead of depending on archive, this target just uses the unarchived, -+# installed version to generate a new archive. Note that we always regenerate -+# the symlink because the RPM_VERSION doesn't include all the version info -- it -+# can leave of things like -beta, -rc1, etc. -+rpm-amd64-spec: rpm-build-area install debian/$(FULL_PACKAGE_TITLE).spec.in RELEASE_$(RPM_VERSION)$(RPM_RELEASE_POSTFIX_UNDERSCORE) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/arm64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec -+ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) - +- -.PHONY: muckrake-build-docker -muckrake-build-docker: - @echo "Building CLI binary with Docker image for Muckrake Check..." @@ -284,14 +315,7 @@ - @echo "Clean up container" - docker rm confluent-muckrake-builder-container - @echo "✅ Binary built successfully at dist/confluent_$(GOOS)_$(GOARCH)/confluent" -+rpm-arm64-spec: -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/amd64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec -+ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) - +- -.PHONY: muckrake-upload-s3 -muckrake-upload-s3: - @echo "Uploading CLI binary to S3..." @@ -317,8 +341,7 @@ - @aws s3 cp $(BINARY_PATH) s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY) \ - --acl public-read --region us-west-2 - @echo "✅ Binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY)" -+rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS - +- -.PHONY: muckrake-clean -muckrake-clean: - @echo "Cleaning up Muckrake Check artifacts..." @@ -326,8 +349,5 @@ - @docker rmi -f confluent-muckrake-builder 2>/dev/null || true - @echo "✅ Cleanup complete" \ No newline at end of file -+RPM_BUILDING/%: -+ mkdir -p $@ -+ +RELEASE_%: + echo 0 > $@ From ecac0bcdce8a072087d3af94e9690434aba2b6d5 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Fri, 10 Oct 2025 16:38:26 -0700 Subject: [PATCH 17/20] Update to use stag AWS S3 bucket and separate main & feature branch to different dirs --- .semaphore/semaphore.yml | 22 +- debian/patches/standard_build_layout.patch | 260 ++++++++++----------- 2 files changed, 136 insertions(+), 146 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index bff0302c09..c530c56782 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -100,12 +100,17 @@ blocks: - checkout - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - "export TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" - - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}/${TIMESTAMP}-${SHORT_SHA}" + - | + if [ "$SEMAPHORE_GIT_BRANCH" = "main" ]; then + export BUILD_ID="main/${TIMESTAMP}-${SHORT_SHA}" + else + export BUILD_ID="branches/${SEMAPHORE_GIT_BRANCH}/${TIMESTAMP}-${SHORT_SHA}" + fi - "echo Build ID: $BUILD_ID" - "GOOS=linux GOARCH=amd64 make muckrake-build-docker" - - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" + - "source assume-iam-role arn:aws:iam::237597620434:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_amd64/confluent S3_KEY=confluent_linux_amd64 make muckrake-upload-s3" - - "echo ✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" + - "echo ✅ linux/amd64 binary uploaded to s3://confluent-cli-internal/confluent-cli-staging/$BUILD_ID/" - name: "Muckrake Check linux/arm64" dependencies: [] @@ -119,12 +124,17 @@ blocks: - checkout - "export SHORT_SHA=$(echo $SEMAPHORE_GIT_SHA | cut -c1-8)" - "export TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" - - "export BUILD_ID=${SEMAPHORE_GIT_BRANCH}/${TIMESTAMP}-${SHORT_SHA}" + - | + if [ "$SEMAPHORE_GIT_BRANCH" = "main" ]; then + export BUILD_ID="main/${TIMESTAMP}-${SHORT_SHA}" + else + export BUILD_ID="branches/${SEMAPHORE_GIT_BRANCH}/${TIMESTAMP}-${SHORT_SHA}" + fi - "echo Build ID: $BUILD_ID" - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" - - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" + - "source assume-iam-role arn:aws:iam::237597620434:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_arm64/confluent S3_KEY=confluent_linux_arm64 make muckrake-upload-s3" - - "echo ✅ linux/arm64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" + - "echo ✅ linux/arm64 binary uploaded to s3://confluent-cli-internal/confluent-cli-staging/$BUILD_ID/" after_pipeline: task: diff --git a/debian/patches/standard_build_layout.patch b/debian/patches/standard_build_layout.patch index f3497f43c4..fa6a42a345 100644 --- a/debian/patches/standard_build_layout.patch +++ b/debian/patches/standard_build_layout.patch @@ -1,5 +1,5 @@ ---- cli/Makefile 2025-10-09 16:20:10.591464253 -0700 -+++ debian/Makefile 2025-10-08 10:35:28.087163073 -0700 +--- cli/Makefile 2025-10-10 16:34:44 ++++ debian/Makefile 2025-09-23 10:17:30 @@ -1,195 +1,163 @@ -SHELL := /bin/bash -GORELEASER_VERSION := v1.21.2 @@ -19,8 +19,10 @@ - endif -else # darwin - $(MAKE) cli-builder --endif -- ++ifndef VERSION ++ VERSION=$(CLI_VERSION) + endif + -# Cross-compile from darwin to any of the OS/Arch pairs below -.PHONY: cross-build -cross-build: @@ -30,7 +32,25 @@ - else # darwin/arm64 - $(MAKE) cli-builder - endif --else ++export PACKAGE_TITLE=cli ++export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) ++export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) ++ ++# Defaults that are likely to vary by platform. These are cleanly separated so ++# it should be easy to maintain altered values on platform-specific branches ++# when the values aren't overridden by the script invoking the Makefile ++ ++APPLY_PATCHES?=yes ++ ++# DESTDIR may be overridden by e.g. debian packaging ++ifeq ($(DESTDIR),) ++DESTDIR=$(CURDIR)/BUILD/ ++endif ++ ++ifeq ($(PACKAGE_TYPE),archive) ++PREFIX=$(PACKAGE_NAME) ++SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) + else - ifeq ($(GOOS),windows) # windows/amd64 - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CGO_LDFLAGS="-fstack-protector -static" $(MAKE) cli-builder - else ifeq ($(GOOS),linux) # linux/amd64 @@ -38,12 +58,17 @@ - else # darwin/amd64 - $(MAKE) cli-builder - endif --endif -- ++PREFIX=/usr ++SYSCONFDIR=/etc/$(PACKAGE_TITLE) + endif + -.PHONY: cli-builder -cli-builder: - GOOS="" GOARCH="" CC="" CXX="" CGO_LDFLAGS="" go install github.com/goreleaser/goreleaser@$(GORELEASER_VERSION) -- ++_empty := ++_space := $(_empty) $(empty) ++split_version := $(subst .,$(_space),$(CLI_VERSION)) + -ifeq ($(GOLANG_FIPS),1) - wget "https://go.dev/dl/go$$(cat .go-version).src.tar.gz" && \ - tar -xf go$$(cat .go-version).src.tar.gz && \ @@ -60,37 +85,6 @@ - rm -rf go go-openssl go$$(cat .go-version).src.tar.gz -else - TAGS=$(TAGS) CC=$(CC) CXX=$(CXX) CGO_LDFLAGS=$(CGO_LDFLAGS) goreleaser build --clean --single-target --snapshot -+ifndef VERSION -+ VERSION=$(CLI_VERSION) - endif - -+export PACKAGE_TITLE=cli -+export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) -+export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) -+ -+# Defaults that are likely to vary by platform. These are cleanly separated so -+# it should be easy to maintain altered values on platform-specific branches -+# when the values aren't overridden by the script invoking the Makefile -+ -+APPLY_PATCHES?=yes -+ -+# DESTDIR may be overridden by e.g. debian packaging -+ifeq ($(DESTDIR),) -+DESTDIR=$(CURDIR)/BUILD/ -+endif -+ -+ifeq ($(PACKAGE_TYPE),archive) -+PREFIX=$(PACKAGE_NAME) -+SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) -+else -+PREFIX=/usr -+SYSCONFDIR=/etc/$(PACKAGE_TITLE) -+endif -+ -+_empty := -+_space := $(_empty) $(empty) -+split_version := $(subst .,$(_space),$(CLI_VERSION)) -+ +all: install + +archive: install @@ -101,12 +95,12 @@ +ifeq ($(APPLY_PATCHES),yes) + git reset --hard HEAD + cat debian/patches/series | xargs -iPATCH bash -c 'patch -p1 < debian/patches/PATCH' -+endif -+ + endif + +BINPATH=$(PREFIX)/bin +LIBPATH=$(PREFIX)/libexec/$(PACKAGE_TITLE) +DOCPATH=$(PREFIX)/share/doc/$(PACKAGE_TITLE) -+ + +# Notes on the archive download links: +# As of v3.0.0, the version number no longer has the 'v' prefix +# As of v3.43.0, we download the '_disableupdates' variant @@ -114,15 +108,27 @@ +install: apply-patches + rm -rf $(DESTDIR)$(PREFIX) + mkdir -p $(DESTDIR)$(PREFIX) -+ + +-.PHONY: clean +-clean: +- for dir in bin dist docs legal prebuilt release-notes; do \ +- [ -d $$dir ] && rm -r $$dir || true; \ +- done + mkdir -p $(DESTDIR)$(BINPATH) + mkdir -p $(DESTDIR)$(LIBPATH) + mkdir -p $(DESTDIR)$(DOCPATH) + mkdir -p $(DESTDIR)$(SYSCONFDIR) -+ + +-.PHONY: lint +-lint: lint-go lint-cli + cp packaging/confluent.sh $(DESTDIR)$(BINPATH)/confluent + chmod 755 $(DESTDIR)$(BINPATH)/confluent -+ + +-.PHONY: lint-go +-lint-go: +- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ +- golangci-lint run --timeout 10m +- @echo "✅ golangci-lint" + cd $(DESTDIR)$(LIBPATH); \ + v=""; if [[ $(word 1,$(split_version)) -eq 2 ]]; then v="v"; fi; \ + disableupdates=""; if [[ $(word 1,$(split_version)) -ge 3 && $(word 2,$(split_version)) -ge 43 ]]; then disableupdates="_disableupdates"; fi; \ @@ -142,38 +148,25 @@ + filepath=windows_amd64/confluent.exe; \ + curl -fs https://$${baseurl}/confluent-cli/binaries/$(CLI_VERSION)/confluent$${version}_windows_amd64.exe -o $${filepath}; \ + chmod 755 $${filepath} -+ -+ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE -+ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt - -+ chown -R root:root $(DESTDIR)$(PREFIX) --.PHONY: clean - clean: -- for dir in bin dist docs legal prebuilt release-notes; do \ -- [ -d $$dir ] && rm -r $$dir || true; \ -- done -- --.PHONY: lint --lint: lint-go lint-cli -- --.PHONY: lint-go --lint-go: -- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ -- golangci-lint run --timeout 10m -- @echo "✅ golangci-lint" -- -.PHONY: lint-cli -lint-cli: cmd/lint/en_US.aff cmd/lint/en_US.dic - go run cmd/lint/main.go -aff-file $(word 1,$^) -dic-file $(word 2,$^) $(ARGS) - @echo "✅ cmd/lint/main.go" -- ++ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE ++ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt + -cmd/lint/en_US.aff: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.aff?format=TEXT" | base64 -D > $@ -- ++ chown -R root:root $(DESTDIR)$(PREFIX) + -cmd/lint/en_US.dic: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.dic?format=TEXT" | base64 -D > $@ -- ++clean: ++ rm -rf $(CURDIR)/$(PACKAGE_NAME)* ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm ++ rm -rf RPM_BUILDING + -.PHONY: unit-test -unit-test: -ifdef CI @@ -181,71 +174,11 @@ - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v -race -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) -else - go test -timeout 0 -v -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) $(UNIT_TEST_ARGS) --endif -+ rm -rf $(CURDIR)/$(PACKAGE_NAME)* -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm -+ rm -rf RPM_BUILDING -+ +distclean: clean +ifneq ($(PACKAGE_TYPE),deb) + git reset --hard HEAD + git status --ignored --porcelain | cut -d ' ' -f 2 | xargs rm -rf -+endif -+ -+RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//' -e 's/-cp[0-9]*//' -e 's/-hotfix[0-9]*//' -e 's/-[0-9]*//') -+ -+# Get any -alpha, -beta (preview), -rc (release candidate), -SNAPSHOT (nightly), -cp (confluent patch), -hotfix piece that we need to put into the Release part of -+# the version since RPM versions don't support non-numeric -+# characters. Ultimately, for something like 0.8.2-beta, we want to end up with -+# Version=0.8.2 Release=0.X.beta -+# where X is the RPM release # of 0.8.2-beta (the prefix 0. forces this to be -+# considered earlier than any 0.8.2 final releases since those will start with -+# Version=0.8.2 Release=1) -+RPM_RELEASE_POSTFIX=$(subst -,,$(subst $(RPM_VERSION),,$(VERSION))) -+ifneq ($(RPM_RELEASE_POSTFIX),) -+ RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) -+ RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) -+else -+ RPM_RELEASE_ID=$(REVISION) -+endif -+ -+rpm: rpm-amd64-build rpm-arm64-build -+ -+rpm-amd64-build: rpm-amd64-spec -+ echo "Building the amd64 RPM" -+ rpmbuild --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz -+ -+rpm-arm64-build: rpm-arm64-spec -+ echo "Building the arm64 RPM" -+ rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz -+ find RPM_BUILDING/{,S}RPMS/ -type f | xargs -n1 -iXXX mv XXX . -+ echo -+ echo "=================================================" -+ echo "The RPMs have been created and can be found here:" -+ @ls -laF $(FULL_PACKAGE_TITLE)*rpm -+ echo "=================================================" -+ -+# Unfortunately, because of version naming issues and the way rpmbuild expects -+# the paths in the tar file to be named, we need to rearchive the package. So -+# instead of depending on archive, this target just uses the unarchived, -+# installed version to generate a new archive. Note that we always regenerate -+# the symlink because the RPM_VERSION doesn't include all the version info -- it -+# can leave of things like -beta, -rc1, etc. -+rpm-amd64-spec: rpm-build-area install debian/$(FULL_PACKAGE_TITLE).spec.in RELEASE_$(RPM_VERSION)$(RPM_RELEASE_POSTFIX_UNDERSCORE) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/arm64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec -+ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ -+rpm-arm64-spec: -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/amd64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec -+ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + endif -.PHONY: build-for-integration-test -build-for-integration-test: @@ -254,17 +187,27 @@ -else - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent ./cmd/confluent -endif -+rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS ++RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//' -e 's/-cp[0-9]*//' -e 's/-hotfix[0-9]*//' -e 's/-[0-9]*//') -.PHONY: build-for-integration-test-windows -build-for-integration-test-windows: -ifdef CI - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent --else ++# Get any -alpha, -beta (preview), -rc (release candidate), -SNAPSHOT (nightly), -cp (confluent patch), -hotfix piece that we need to put into the Release part of ++# the version since RPM versions don't support non-numeric ++# characters. Ultimately, for something like 0.8.2-beta, we want to end up with ++# Version=0.8.2 Release=0.X.beta ++# where X is the RPM release # of 0.8.2-beta (the prefix 0. forces this to be ++# considered earlier than any 0.8.2 final releases since those will start with ++# Version=0.8.2 Release=1) ++RPM_RELEASE_POSTFIX=$(subst -,,$(subst $(RPM_VERSION),,$(VERSION))) ++ifneq ($(RPM_RELEASE_POSTFIX),) ++ RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) ++ RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) + else - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent --endif -+RPM_BUILDING/%: -+ mkdir -p $@ ++ RPM_RELEASE_ID=$(REVISION) + endif -.PHONY: integration-test -integration-test: @@ -280,14 +223,27 @@ - go test -timeout 0 -v $$(go list ./... | grep github.com/confluentinc/cli/v4/test) $(INTEGRATION_TEST_ARGS) && \ - go tool covdata textfmt -i $${GOCOVERDIR} -o coverage.integration.out -endif -- ++rpm: rpm-amd64-build rpm-arm64-build + -.PHONY: test -test: unit-test integration-test -- ++rpm-amd64-build: rpm-amd64-spec ++ echo "Building the amd64 RPM" ++ rpmbuild --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz + -.PHONY: generate-packaging-patch -generate-packaging-patch: - diff -u Makefile debian/Makefile | sed "1 s_Makefile_cli/Makefile_" > debian/patches/standard_build_layout.patch -- ++rpm-arm64-build: rpm-arm64-spec ++ echo "Building the arm64 RPM" ++ rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz ++ find RPM_BUILDING/{,S}RPMS/ -type f | xargs -n1 -iXXX mv XXX . ++ echo ++ echo "=================================================" ++ echo "The RPMs have been created and can be found here:" ++ @ls -laF $(FULL_PACKAGE_TITLE)*rpm ++ echo "=================================================" + -.PHONY: coverage -coverage: ## Merge coverage data from unit and integration tests into coverage.txt - @echo "Merging coverage data..." @@ -296,7 +252,20 @@ - @tail -n +2 coverage.integration.out >> coverage.txt - @echo "Coverage data saved to: coverage.txt" - @artifact push workflow coverage.txt -- ++# Unfortunately, because of version naming issues and the way rpmbuild expects ++# the paths in the tar file to be named, we need to rearchive the package. So ++# instead of depending on archive, this target just uses the unarchived, ++# installed version to generate a new archive. Note that we always regenerate ++# the symlink because the RPM_VERSION doesn't include all the version info -- it ++# can leave of things like -beta, -rc1, etc. ++rpm-amd64-spec: rpm-build-area install debian/$(FULL_PACKAGE_TITLE).spec.in RELEASE_$(RPM_VERSION)$(RPM_RELEASE_POSTFIX_UNDERSCORE) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/arm64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec ++ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + -.PHONY: muckrake-build-docker -muckrake-build-docker: - @echo "Building CLI binary with Docker image for Muckrake Check..." @@ -315,7 +284,14 @@ - @echo "Clean up container" - docker rm confluent-muckrake-builder-container - @echo "✅ Binary built successfully at dist/confluent_$(GOOS)_$(GOARCH)/confluent" -- ++rpm-arm64-spec: ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/amd64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec ++ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + -.PHONY: muckrake-upload-s3 -muckrake-upload-s3: - @echo "Uploading CLI binary to S3..." @@ -341,7 +317,8 @@ - @aws s3 cp $(BINARY_PATH) s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY) \ - --acl public-read --region us-west-2 - @echo "✅ Binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY)" -- ++rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS + -.PHONY: muckrake-clean -muckrake-clean: - @echo "Cleaning up Muckrake Check artifacts..." @@ -349,5 +326,8 @@ - @docker rmi -f confluent-muckrake-builder 2>/dev/null || true - @echo "✅ Cleanup complete" \ No newline at end of file ++RPM_BUILDING/%: ++ mkdir -p $@ ++ +RELEASE_%: + echo 0 > $@ From 2e0194a6116d9d717773106dc25ec077c3cacb1c Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Sat, 11 Oct 2025 22:30:45 -0700 Subject: [PATCH 18/20] Revert to use prod AWS S3 bucket --- .semaphore/semaphore.yml | 8 ++++---- debian/patches/standard_build_layout.patch | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index c530c56782..17cf340232 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -108,9 +108,9 @@ blocks: fi - "echo Build ID: $BUILD_ID" - "GOOS=linux GOARCH=amd64 make muckrake-build-docker" - - "source assume-iam-role arn:aws:iam::237597620434:role/cli-release-semaphore" + - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_amd64/confluent S3_KEY=confluent_linux_amd64 make muckrake-upload-s3" - - "echo ✅ linux/amd64 binary uploaded to s3://confluent-cli-internal/confluent-cli-staging/$BUILD_ID/" + - "echo ✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" - name: "Muckrake Check linux/arm64" dependencies: [] @@ -132,9 +132,9 @@ blocks: fi - "echo Build ID: $BUILD_ID" - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" - - "source assume-iam-role arn:aws:iam::237597620434:role/cli-release-semaphore" + - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_arm64/confluent S3_KEY=confluent_linux_arm64 make muckrake-upload-s3" - - "echo ✅ linux/arm64 binary uploaded to s3://confluent-cli-internal/confluent-cli-staging/$BUILD_ID/" + - "echo ✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" after_pipeline: task: diff --git a/debian/patches/standard_build_layout.patch b/debian/patches/standard_build_layout.patch index fa6a42a345..f98db0ca42 100644 --- a/debian/patches/standard_build_layout.patch +++ b/debian/patches/standard_build_layout.patch @@ -1,4 +1,4 @@ ---- cli/Makefile 2025-10-10 16:34:44 +--- cli/Makefile 2025-10-10 16:36:23 +++ debian/Makefile 2025-09-23 10:17:30 @@ -1,195 +1,163 @@ -SHELL := /bin/bash From 7e1d58e71afe4c1b751d11b73221471cb6b47b60 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Sat, 11 Oct 2025 22:34:21 -0700 Subject: [PATCH 19/20] Fix typo --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 17cf340232..6c92f61a59 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -134,7 +134,7 @@ blocks: - "GOOS=linux GOARCH=arm64 make muckrake-build-docker" - "source assume-iam-role arn:aws:iam::050879227952:role/cli-release-semaphore" - "BUILD_ID=$BUILD_ID BINARY_PATH=dist/confluent_linux_arm64/confluent S3_KEY=confluent_linux_arm64 make muckrake-upload-s3" - - "echo ✅ linux/amd64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" + - "echo ✅ linux/arm64 binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$BUILD_ID/" after_pipeline: task: From c1a9aaa1f4e03d022eaf5ad1c3b264ff9186b07b Mon Sep 17 00:00:00 2001 From: Steven Gagniere Date: Tue, 14 Oct 2025 11:33:14 -0700 Subject: [PATCH 20/20] make generate packaging patch --- debian/patches/standard_build_layout.patch | 260 +++++++++++---------- 1 file changed, 140 insertions(+), 120 deletions(-) diff --git a/debian/patches/standard_build_layout.patch b/debian/patches/standard_build_layout.patch index f98db0ca42..133906288b 100644 --- a/debian/patches/standard_build_layout.patch +++ b/debian/patches/standard_build_layout.patch @@ -1,5 +1,5 @@ ---- cli/Makefile 2025-10-10 16:36:23 -+++ debian/Makefile 2025-09-23 10:17:30 +--- cli/Makefile 2025-10-14 11:32:51.152782132 -0700 ++++ debian/Makefile 2025-10-10 18:28:49.581238914 -0700 @@ -1,195 +1,163 @@ -SHELL := /bin/bash -GORELEASER_VERSION := v1.21.2 @@ -19,10 +19,8 @@ - endif -else # darwin - $(MAKE) cli-builder -+ifndef VERSION -+ VERSION=$(CLI_VERSION) - endif - +-endif +- -# Cross-compile from darwin to any of the OS/Arch pairs below -.PHONY: cross-build -cross-build: @@ -32,25 +30,7 @@ - else # darwin/arm64 - $(MAKE) cli-builder - endif -+export PACKAGE_TITLE=cli -+export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) -+export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) -+ -+# Defaults that are likely to vary by platform. These are cleanly separated so -+# it should be easy to maintain altered values on platform-specific branches -+# when the values aren't overridden by the script invoking the Makefile -+ -+APPLY_PATCHES?=yes -+ -+# DESTDIR may be overridden by e.g. debian packaging -+ifeq ($(DESTDIR),) -+DESTDIR=$(CURDIR)/BUILD/ -+endif -+ -+ifeq ($(PACKAGE_TYPE),archive) -+PREFIX=$(PACKAGE_NAME) -+SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) - else +-else - ifeq ($(GOOS),windows) # windows/amd64 - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CGO_LDFLAGS="-fstack-protector -static" $(MAKE) cli-builder - else ifeq ($(GOOS),linux) # linux/amd64 @@ -58,17 +38,12 @@ - else # darwin/amd64 - $(MAKE) cli-builder - endif -+PREFIX=/usr -+SYSCONFDIR=/etc/$(PACKAGE_TITLE) - endif - +-endif +- -.PHONY: cli-builder -cli-builder: - GOOS="" GOARCH="" CC="" CXX="" CGO_LDFLAGS="" go install github.com/goreleaser/goreleaser@$(GORELEASER_VERSION) -+_empty := -+_space := $(_empty) $(empty) -+split_version := $(subst .,$(_space),$(CLI_VERSION)) - +- -ifeq ($(GOLANG_FIPS),1) - wget "https://go.dev/dl/go$$(cat .go-version).src.tar.gz" && \ - tar -xf go$$(cat .go-version).src.tar.gz && \ @@ -85,6 +60,37 @@ - rm -rf go go-openssl go$$(cat .go-version).src.tar.gz -else - TAGS=$(TAGS) CC=$(CC) CXX=$(CXX) CGO_LDFLAGS=$(CGO_LDFLAGS) goreleaser build --clean --single-target --snapshot ++ifndef VERSION ++ VERSION=$(CLI_VERSION) + endif + ++export PACKAGE_TITLE=cli ++export FULL_PACKAGE_TITLE=confluent-$(PACKAGE_TITLE) ++export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) ++ ++# Defaults that are likely to vary by platform. These are cleanly separated so ++# it should be easy to maintain altered values on platform-specific branches ++# when the values aren't overridden by the script invoking the Makefile ++ ++APPLY_PATCHES?=yes ++ ++# DESTDIR may be overridden by e.g. debian packaging ++ifeq ($(DESTDIR),) ++DESTDIR=$(CURDIR)/BUILD/ ++endif ++ ++ifeq ($(PACKAGE_TYPE),archive) ++PREFIX=$(PACKAGE_NAME) ++SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) ++else ++PREFIX=/usr ++SYSCONFDIR=/etc/$(PACKAGE_TITLE) ++endif ++ ++_empty := ++_space := $(_empty) $(empty) ++split_version := $(subst .,$(_space),$(CLI_VERSION)) ++ +all: install + +archive: install @@ -95,12 +101,12 @@ +ifeq ($(APPLY_PATCHES),yes) + git reset --hard HEAD + cat debian/patches/series | xargs -iPATCH bash -c 'patch -p1 < debian/patches/PATCH' - endif - ++endif ++ +BINPATH=$(PREFIX)/bin +LIBPATH=$(PREFIX)/libexec/$(PACKAGE_TITLE) +DOCPATH=$(PREFIX)/share/doc/$(PACKAGE_TITLE) - ++ +# Notes on the archive download links: +# As of v3.0.0, the version number no longer has the 'v' prefix +# As of v3.43.0, we download the '_disableupdates' variant @@ -108,27 +114,15 @@ +install: apply-patches + rm -rf $(DESTDIR)$(PREFIX) + mkdir -p $(DESTDIR)$(PREFIX) - --.PHONY: clean --clean: -- for dir in bin dist docs legal prebuilt release-notes; do \ -- [ -d $$dir ] && rm -r $$dir || true; \ -- done ++ + mkdir -p $(DESTDIR)$(BINPATH) + mkdir -p $(DESTDIR)$(LIBPATH) + mkdir -p $(DESTDIR)$(DOCPATH) + mkdir -p $(DESTDIR)$(SYSCONFDIR) - --.PHONY: lint --lint: lint-go lint-cli ++ + cp packaging/confluent.sh $(DESTDIR)$(BINPATH)/confluent + chmod 755 $(DESTDIR)$(BINPATH)/confluent - --.PHONY: lint-go --lint-go: -- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ -- golangci-lint run --timeout 10m -- @echo "✅ golangci-lint" ++ + cd $(DESTDIR)$(LIBPATH); \ + v=""; if [[ $(word 1,$(split_version)) -eq 2 ]]; then v="v"; fi; \ + disableupdates=""; if [[ $(word 1,$(split_version)) -ge 3 && $(word 2,$(split_version)) -ge 43 ]]; then disableupdates="_disableupdates"; fi; \ @@ -148,25 +142,38 @@ + filepath=windows_amd64/confluent.exe; \ + curl -fs https://$${baseurl}/confluent-cli/binaries/$(CLI_VERSION)/confluent$${version}_windows_amd64.exe -o $${filepath}; \ + chmod 755 $${filepath} ++ ++ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE ++ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt + ++ chown -R root:root $(DESTDIR)$(PREFIX) +-.PHONY: clean + clean: +- for dir in bin dist docs legal prebuilt release-notes; do \ +- [ -d $$dir ] && rm -r $$dir || true; \ +- done +- +-.PHONY: lint +-lint: lint-go lint-cli +- +-.PHONY: lint-go +-lint-go: +- go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.8 && \ +- golangci-lint run --timeout 10m +- @echo "✅ golangci-lint" +- -.PHONY: lint-cli -lint-cli: cmd/lint/en_US.aff cmd/lint/en_US.dic - go run cmd/lint/main.go -aff-file $(word 1,$^) -dic-file $(word 2,$^) $(ARGS) - @echo "✅ cmd/lint/main.go" -+ cp LICENSE $(DESTDIR)$(DOCPATH)/LICENSE -+ $(DESTDIR)$(BINPATH)/confluent --version | awk -F' ' '{ print $3 }' > $(DESTDIR)$(DOCPATH)/version.txt - +- -cmd/lint/en_US.aff: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.aff?format=TEXT" | base64 -D > $@ -+ chown -R root:root $(DESTDIR)$(PREFIX) - +- -cmd/lint/en_US.dic: - curl -s "https://chromium.googlesource.com/chromium/deps/hunspell_dictionaries/+/master/en_US.dic?format=TEXT" | base64 -D > $@ -+clean: -+ rm -rf $(CURDIR)/$(PACKAGE_NAME)* -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm -+ rm -rf RPM_BUILDING - +- -.PHONY: unit-test -unit-test: -ifdef CI @@ -174,11 +181,71 @@ - gotestsum --junitfile unit-test-report.xml -- -timeout 0 -v -race -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) -else - go test -timeout 0 -v -coverprofile=coverage.unit.out -covermode=atomic $$(go list ./... | grep -v github.com/confluentinc/cli/v4/test) $(UNIT_TEST_ARGS) +-endif ++ rm -rf $(CURDIR)/$(PACKAGE_NAME)* ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm ++ rm -rf RPM_BUILDING ++ +distclean: clean +ifneq ($(PACKAGE_TYPE),deb) + git reset --hard HEAD + git status --ignored --porcelain | cut -d ' ' -f 2 | xargs rm -rf - endif ++endif ++ ++RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//' -e 's/-cp[0-9]*//' -e 's/-hotfix[0-9]*//' -e 's/-[0-9]*//') ++ ++# Get any -alpha, -beta (preview), -rc (release candidate), -SNAPSHOT (nightly), -cp (confluent patch), -hotfix piece that we need to put into the Release part of ++# the version since RPM versions don't support non-numeric ++# characters. Ultimately, for something like 0.8.2-beta, we want to end up with ++# Version=0.8.2 Release=0.X.beta ++# where X is the RPM release # of 0.8.2-beta (the prefix 0. forces this to be ++# considered earlier than any 0.8.2 final releases since those will start with ++# Version=0.8.2 Release=1) ++RPM_RELEASE_POSTFIX=$(subst -,,$(subst $(RPM_VERSION),,$(VERSION))) ++ifneq ($(RPM_RELEASE_POSTFIX),) ++ RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) ++ RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) ++else ++ RPM_RELEASE_ID=$(REVISION) ++endif ++ ++rpm: rpm-amd64-build rpm-arm64-build ++ ++rpm-amd64-build: rpm-amd64-spec ++ echo "Building the amd64 RPM" ++ rpmbuild --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz ++ ++rpm-arm64-build: rpm-arm64-spec ++ echo "Building the arm64 RPM" ++ rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz ++ find RPM_BUILDING/{,S}RPMS/ -type f | xargs -n1 -iXXX mv XXX . ++ echo ++ echo "=================================================" ++ echo "The RPMs have been created and can be found here:" ++ @ls -laF $(FULL_PACKAGE_TITLE)*rpm ++ echo "=================================================" ++ ++# Unfortunately, because of version naming issues and the way rpmbuild expects ++# the paths in the tar file to be named, we need to rearchive the package. So ++# instead of depending on archive, this target just uses the unarchived, ++# installed version to generate a new archive. Note that we always regenerate ++# the symlink because the RPM_VERSION doesn't include all the version info -- it ++# can leave of things like -beta, -rc1, etc. ++rpm-amd64-spec: rpm-build-area install debian/$(FULL_PACKAGE_TITLE).spec.in RELEASE_$(RPM_VERSION)$(RPM_RELEASE_POSTFIX_UNDERSCORE) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/arm64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec ++ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ ++rpm-arm64-spec: ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/amd64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec ++ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) ++ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -.PHONY: build-for-integration-test -build-for-integration-test: @@ -187,27 +254,17 @@ -else - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent ./cmd/confluent -endif -+RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//' -e 's/-cp[0-9]*//' -e 's/-hotfix[0-9]*//' -e 's/-[0-9]*//') ++rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS -.PHONY: build-for-integration-test-windows -build-for-integration-test-windows: -ifdef CI - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent -+# Get any -alpha, -beta (preview), -rc (release candidate), -SNAPSHOT (nightly), -cp (confluent patch), -hotfix piece that we need to put into the Release part of -+# the version since RPM versions don't support non-numeric -+# characters. Ultimately, for something like 0.8.2-beta, we want to end up with -+# Version=0.8.2 Release=0.X.beta -+# where X is the RPM release # of 0.8.2-beta (the prefix 0. forces this to be -+# considered earlier than any 0.8.2 final releases since those will start with -+# Version=0.8.2 Release=1) -+RPM_RELEASE_POSTFIX=$(subst -,,$(subst $(RPM_VERSION),,$(VERSION))) -+ifneq ($(RPM_RELEASE_POSTFIX),) -+ RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) -+ RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) - else +-else - go build -cover -ldflags="-s -w -X main.commit="00000000" -X main.date="1970-01-01T00:00:00Z" -X main.isTest=true" -o test/bin/confluent.exe ./cmd/confluent -+ RPM_RELEASE_ID=$(REVISION) - endif +-endif ++RPM_BUILDING/%: ++ mkdir -p $@ -.PHONY: integration-test -integration-test: @@ -223,27 +280,14 @@ - go test -timeout 0 -v $$(go list ./... | grep github.com/confluentinc/cli/v4/test) $(INTEGRATION_TEST_ARGS) && \ - go tool covdata textfmt -i $${GOCOVERDIR} -o coverage.integration.out -endif -+rpm: rpm-amd64-build rpm-arm64-build - +- -.PHONY: test -test: unit-test integration-test -+rpm-amd64-build: rpm-amd64-spec -+ echo "Building the amd64 RPM" -+ rpmbuild --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz - +- -.PHONY: generate-packaging-patch -generate-packaging-patch: - diff -u Makefile debian/Makefile | sed "1 s_Makefile_cli/Makefile_" > debian/patches/standard_build_layout.patch -+rpm-arm64-build: rpm-arm64-spec -+ echo "Building the arm64 RPM" -+ rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" --nodebuginfo -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz || rpmbuild --target=aarch64 --define="_topdir `pwd`/RPM_BUILDING" -tb RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz -+ find RPM_BUILDING/{,S}RPMS/ -type f | xargs -n1 -iXXX mv XXX . -+ echo -+ echo "=================================================" -+ echo "The RPMs have been created and can be found here:" -+ @ls -laF $(FULL_PACKAGE_TITLE)*rpm -+ echo "=================================================" - +- -.PHONY: coverage -coverage: ## Merge coverage data from unit and integration tests into coverage.txt - @echo "Merging coverage data..." @@ -252,20 +296,7 @@ - @tail -n +2 coverage.integration.out >> coverage.txt - @echo "Coverage data saved to: coverage.txt" - @artifact push workflow coverage.txt -+# Unfortunately, because of version naming issues and the way rpmbuild expects -+# the paths in the tar file to be named, we need to rearchive the package. So -+# instead of depending on archive, this target just uses the unarchived, -+# installed version to generate a new archive. Note that we always regenerate -+# the symlink because the RPM_VERSION doesn't include all the version info -- it -+# can leave of things like -beta, -rc1, etc. -+rpm-amd64-spec: rpm-build-area install debian/$(FULL_PACKAGE_TITLE).spec.in RELEASE_$(RPM_VERSION)$(RPM_RELEASE_POSTFIX_UNDERSCORE) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/arm64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec -+ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) - +- -.PHONY: muckrake-build-docker -muckrake-build-docker: - @echo "Building CLI binary with Docker image for Muckrake Check..." @@ -284,14 +315,7 @@ - @echo "Clean up container" - docker rm confluent-muckrake-builder-container - @echo "✅ Binary built successfully at dist/confluent_$(GOOS)_$(GOARCH)/confluent" -+rpm-arm64-spec: -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ sed "s/##RPMVERSION##/$(RPM_VERSION)/g; s/##RPMRELEASE##/$(RPM_RELEASE_ID)/g; s/##EXCLUDEARCH##/amd64/g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec -+ rm -f $@ && tar -czf RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) -+ rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) - +- -.PHONY: muckrake-upload-s3 -muckrake-upload-s3: - @echo "Uploading CLI binary to S3..." @@ -317,8 +341,7 @@ - @aws s3 cp $(BINARY_PATH) s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY) \ - --acl public-read --region us-west-2 - @echo "✅ Binary uploaded to s3://confluent.cloud/confluent-cli/development-builds/$(BUILD_ID)/$(S3_KEY)" -+rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS - +- -.PHONY: muckrake-clean -muckrake-clean: - @echo "Cleaning up Muckrake Check artifacts..." @@ -326,8 +349,5 @@ - @docker rmi -f confluent-muckrake-builder 2>/dev/null || true - @echo "✅ Cleanup complete" \ No newline at end of file -+RPM_BUILDING/%: -+ mkdir -p $@ -+ +RELEASE_%: + echo 0 > $@