From 7506019f734c79c6b49a6cbda251fcdad3ccec24 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:54:53 +0100 Subject: [PATCH 01/12] add Spark 4.1.1 --- spark-k8s/Dockerfile | 11 ++++-- spark-k8s/boil-config.toml | 18 +++++++++ .../4.1.1/0001-Update-CycloneDX-plugin.patch | 38 +++++++++++++++++++ .../stackable/patches/4.1.1/patchable.toml | 1 + 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 spark-k8s/stackable/patches/4.1.1/0001-Update-CycloneDX-plugin.patch create mode 100644 spark-k8s/stackable/patches/4.1.1/patchable.toml diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 562a435eb..2afbd7a4d 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -272,11 +272,16 @@ WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/ext RUN < +Date: Thu, 15 Jan 2026 14:27:23 +0100 +Subject: Update CycloneDX plugin + +--- + dev/make-distribution.sh | 1 - + pom.xml | 5 +++++ + 2 files changed, 5 insertions(+), 1 deletion(-) + +diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh +index 16598bda873..327fa763144 100755 +--- a/dev/make-distribution.sh ++++ b/dev/make-distribution.sh +@@ -185,7 +185,6 @@ else + -Dmaven.javadoc.skip=true \ + -Dmaven.scaladoc.skip=true \ + -Dmaven.source.skip \ +- -Dcyclonedx.skip=true \ + $@) + fi + +diff --git a/pom.xml b/pom.xml +index dc757d78812..05f1af034f3 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -3333,6 +3333,11 @@ + org.cyclonedx + cyclonedx-maven-plugin + 2.9.1 ++ ++ application ++ 1.5 ++ false ++ + + + package diff --git a/spark-k8s/stackable/patches/4.1.1/patchable.toml b/spark-k8s/stackable/patches/4.1.1/patchable.toml new file mode 100644 index 000000000..15b658491 --- /dev/null +++ b/spark-k8s/stackable/patches/4.1.1/patchable.toml @@ -0,0 +1 @@ +base = "c0690c763bafabd08e7079d1137fa0a769a05bae" From 82f8be3038bfe5ed5e1b5fa51ab746aa8670a82e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:56:46 +0100 Subject: [PATCH 02/12] delete 3.5.6 and 4.0.1 --- spark-k8s/boil-config.toml | 36 -------- .../3.5.6/0001-Update-CycloneDX-plugin.patch | 40 --------- ...1311-BUILD-Promote-bcprov-jdk18on-to.patch | 83 ------------------- .../stackable/patches/3.5.6/patchable.toml | 1 - .../4.0.1/0001-Update-CycloneDX-plugin.patch | 38 --------- .../stackable/patches/4.0.1/patchable.toml | 2 - 6 files changed, 200 deletions(-) delete mode 100644 spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch delete mode 100644 spark-k8s/stackable/patches/3.5.6/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch delete mode 100644 spark-k8s/stackable/patches/3.5.6/patchable.toml delete mode 100644 spark-k8s/stackable/patches/4.0.1/0001-Update-CycloneDX-plugin.patch delete mode 100644 spark-k8s/stackable/patches/4.0.1/patchable.toml diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 8ae2cdece..6cff0d34a 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -1,21 +1,3 @@ -[versions."3.5.6".local-images] -"hadoop/hadoop" = "3.4.2" -java-base = "17" -java-devel = "17" -hbase = "2.6.3" - -[versions."3.5.6".build-arguments] -python-version = "3.11" -aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop -azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop -azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.6 -stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -jmx-exporter-version = "1.3.0" -tini-version = "0.19.0" -hbase-connector-version = "1.0.1" - [versions."3.5.7".local-images] "hadoop/hadoop" = "3.4.2" java-base = "17" @@ -34,24 +16,6 @@ jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1" -[versions."4.0.1".local-images] -"hadoop/hadoop" = "3.4.2" -java-base = "17" -java-devel = "17" -hbase = "2.6.3" - -[versions."4.0.1".build-arguments] -python-version = "3.11" -aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop -azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop -azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.18.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/4.0.1 -stax2-api-version = "4.2.2" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2 -woodstox-core-version = "7.0.0" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2 -jmx-exporter-version = "1.3.0" -tini-version = "0.19.0" -hbase-connector-version = "1.0.1" # This is not supported in Spark 4 yet. - [versions."4.1.1".local-images] "hadoop/hadoop" = "3.4.2" java-base = "21" diff --git a/spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch deleted file mode 100644 index 9ad9eea3c..000000000 --- a/spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch +++ /dev/null @@ -1,40 +0,0 @@ -From 94ccf32b4d0eb7c3191b4e5a646605e7386c39ff Mon Sep 17 00:00:00 2001 -From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Tue, 11 Mar 2025 17:29:39 +0200 -Subject: Update CycloneDX plugin - ---- - dev/make-distribution.sh | 1 - - pom.xml | 7 ++++++- - 2 files changed, 6 insertions(+), 2 deletions(-) - -diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh -index ef7c010e93..0f4c1c74e4 100755 ---- a/dev/make-distribution.sh -+++ b/dev/make-distribution.sh -@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \ - -Dmaven.javadoc.skip=true \ - -Dmaven.scaladoc.skip=true \ - -Dmaven.source.skip \ -- -Dcyclonedx.skip=true \ - $@) - - # Actually build the jar -diff --git a/pom.xml b/pom.xml -index 68e2c422a2..6216ebb08e 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -3534,7 +3534,12 @@ - - org.cyclonedx - cyclonedx-maven-plugin -- 2.7.9 -+ 2.8.0 -+ -+ application -+ 1.5 -+ false -+ - - - package diff --git a/spark-k8s/stackable/patches/3.5.6/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch b/spark-k8s/stackable/patches/3.5.6/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch deleted file mode 100644 index 1a7029a6f..000000000 --- a/spark-k8s/stackable/patches/3.5.6/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch +++ /dev/null @@ -1,83 +0,0 @@ -From 37d866706d952702effd640babf891fef349da7d Mon Sep 17 00:00:00 2001 -From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Tue, 22 Jul 2025 17:34:03 +0200 -Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope - ---- - LICENSE-binary | 1 + - assembly/pom.xml | 14 ++++++++++++++ - dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 + - licenses-binary/LICENSE-bouncycastle.txt | 13 +++++++++++++ - 4 files changed, 29 insertions(+) - create mode 100644 licenses-binary/LICENSE-bouncycastle.txt - -diff --git a/LICENSE-binary b/LICENSE-binary -index 05645977a0..9834cf333f 100644 ---- a/LICENSE-binary -+++ b/LICENSE-binary -@@ -480,6 +480,7 @@ org.typelevel:algebra_2.12:jar - org.typelevel:cats-kernel_2.12 - org.typelevel:machinist_2.12 - net.razorvine:pickle -+org.bouncycastle:bcprov-jdk18on - org.slf4j:jcl-over-slf4j - org.slf4j:jul-to-slf4j - org.slf4j:slf4j-api -diff --git a/assembly/pom.xml b/assembly/pom.xml -index dcc46b0b82..def40ad52e 100644 ---- a/assembly/pom.xml -+++ b/assembly/pom.xml -@@ -85,8 +85,22 @@ - guava - ${hadoop.deps.scope} - -+ -+ -+ -+ org.bouncycastle -+ bcprov-jdk18on -+ ${hadoop.deps.scope} -+ -+ - - -+ - - - -diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 -index dbf0cb34c5..689f50612b 100644 ---- a/dev/deps/spark-deps-hadoop-3-hive-2.3 -+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 -@@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar - azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar - azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar - azure-storage/7.0.1//azure-storage-7.0.1.jar -+bcprov-jdk18on/1.77//bcprov-jdk18on-1.77.jar - blas/3.0.3//blas-3.0.3.jar - bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar - breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar -diff --git a/licenses-binary/LICENSE-bouncycastle.txt b/licenses-binary/LICENSE-bouncycastle.txt -new file mode 100644 -index 0000000000..277dcd1ebb ---- /dev/null -+++ b/licenses-binary/LICENSE-bouncycastle.txt -@@ -0,0 +1,13 @@ -+Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). -+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -+associated documentation files (the "Software"), to deal in the Software without restriction, -+including without limitation the rights to use, copy, modify, merge, publish, distribute, -+sub license, and/or sell copies of the Software, and to permit persons to whom the Software is -+furnished to do so, subject to the following conditions: The above copyright notice and this -+permission notice shall be included in all copies or substantial portions of the Software. -+ -+**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT -+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT -+OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** diff --git a/spark-k8s/stackable/patches/3.5.6/patchable.toml b/spark-k8s/stackable/patches/3.5.6/patchable.toml deleted file mode 100644 index 633d26c88..000000000 --- a/spark-k8s/stackable/patches/3.5.6/patchable.toml +++ /dev/null @@ -1 +0,0 @@ -base = "303c18c74664f161b9b969ac343784c088b47593" diff --git a/spark-k8s/stackable/patches/4.0.1/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/4.0.1/0001-Update-CycloneDX-plugin.patch deleted file mode 100644 index 863f280c6..000000000 --- a/spark-k8s/stackable/patches/4.0.1/0001-Update-CycloneDX-plugin.patch +++ /dev/null @@ -1,38 +0,0 @@ -From b5de94e20aff25a394c6095c0649b4fcbaa941aa Mon Sep 17 00:00:00 2001 -From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Fri, 4 Jul 2025 15:54:55 +0200 -Subject: Update CycloneDX plugin - ---- - dev/make-distribution.sh | 1 - - pom.xml | 5 +++++ - 2 files changed, 5 insertions(+), 1 deletion(-) - -diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh -index 16607e45ae6..44e345a245d 100755 ---- a/dev/make-distribution.sh -+++ b/dev/make-distribution.sh -@@ -176,7 +176,6 @@ BUILD_COMMAND=("$MVN" clean package \ - -Dmaven.javadoc.skip=true \ - -Dmaven.scaladoc.skip=true \ - -Dmaven.source.skip \ -- -Dcyclonedx.skip=true \ - $@) - - # Actually build the jar -diff --git a/pom.xml b/pom.xml -index 22922143fc3..59c3747c625 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -3327,6 +3327,11 @@ - org.cyclonedx - cyclonedx-maven-plugin - 2.8.0 -+ -+ application -+ 1.5 -+ false -+ - - - package diff --git a/spark-k8s/stackable/patches/4.0.1/patchable.toml b/spark-k8s/stackable/patches/4.0.1/patchable.toml deleted file mode 100644 index bd074097c..000000000 --- a/spark-k8s/stackable/patches/4.0.1/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -base = "29434ea766b0fc3c3bf6eaadb43a8f931133649e" -mirror = "https://github.com/stackabletech/spark.git" From 802933cbad77955e12e99036c643f5e99f686e3c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 16 Jan 2026 10:59:47 +0100 Subject: [PATCH 03/12] split Dockerfiles per version --- spark-k8s/{Dockerfile => Dockerfile.3} | 50 +---- spark-k8s/Dockerfile.4 | 263 +++++++++++++++++++++++++ spark-k8s/boil-config.toml | 10 +- 3 files changed, 279 insertions(+), 44 deletions(-) rename spark-k8s/{Dockerfile => Dockerfile.3} (85%) create mode 100644 spark-k8s/Dockerfile.4 diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile.3 similarity index 85% rename from spark-k8s/Dockerfile rename to spark-k8s/Dockerfile.3 index 2afbd7a4d..668242723 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile.3 @@ -62,16 +62,6 @@ COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patche RUN <>> Build spark +RUN <]' '{print $3}') + + mkdir -p dist/connect + cd dist/connect + + case "${PRODUCT_VERSION}" in + 4*) + cp "/stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/sql/connect/server/target/spark-connect_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" . + cp "/stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/sql/connect/common/target/spark-connect-common_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" . + cp "/stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/sql/connect/client/jvm/target/spark-connect-client-jvm_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" . + ;; + *) + cp "/stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/connector/connect/server/target/spark-connect_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" . + cp "/stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/connector/connect/common/target/spark-connect-common_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" . + cp "/stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/connector/connect/client/jvm/target/spark-connect-client-jvm_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" . + ;; + esac + + # This link is needed by the operator and is kept for backwards compatibility. + # TODO: remove it at some time in the future. + ln -s "spark-connect_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" "spark-connect_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}.jar" + # Link to the spark-connect jar without the stackable suffix and scala version. + # This link supersedes the previous link. + ln -s "spark-connect_${SCALA_BINARY_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.jar" "spark-connect-${PRODUCT_VERSION}.jar" +EOF + +# <<< Build spark + +WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/jars + +# Copy modules required for s3a:// +COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \ + /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \ + /stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar \ + ./ + +# Copy modules required for abfs:// +COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \ + /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \ + /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar \ + /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar \ + ./ + +COPY spark-k8s/stackable/jmx /stackable/jmx + +WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars + +RUN < Date: Fri, 16 Jan 2026 11:03:08 +0100 Subject: [PATCH 04/12] update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1bd5074..d79133bdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file. - opensearch-dashboards: Add `3.4.0` ([#1392]). - testing-tools: build testing tools subimages in workflow ([#1366]). - kafka: Add `4.1.1` ([#1395]). +- spark: Add `4.1.1`, use one Dockerfile per version and remove all HBase deps from the Spark4 image ([#1402]). ### Changed @@ -33,6 +34,7 @@ All notable changes to this project will be documented in this file. - superset: Remove 4.0.2 and 4.1.2 ([#1394]). - kafka: Remove `3.7.2` and `4.1.0` ([#1395]). - opa: remove 1.4.2 ([#1396]). +- spark: Remove `3.5.6` and `4.0.1` ([#1402]). ### Fixed @@ -60,6 +62,7 @@ All notable changes to this project will be documented in this file. [#1394]: https://github.com/stackabletech/docker-images/pull/1394 [#1395]: https://github.com/stackabletech/docker-images/pull/1395 [#1396]: https://github.com/stackabletech/docker-images/pull/1396 +[#1402]: https://github.com/stackabletech/docker-images/pull/1402 ## [25.11.0] - 2025-11-07 From 73930cb07c404a6316d8ebac4c09a6c62f16624e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 16 Jan 2026 12:14:01 +0100 Subject: [PATCH 05/12] cleanup and successful build --- CHANGELOG.md | 5 +++- spark-k8s/Dockerfile.3 | 14 +++++----- spark-k8s/Dockerfile.4 | 58 ++++++++++++++---------------------------- 3 files changed, 30 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d79133bdc..9c0021eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ All notable changes to this project will be documented in this file. - opensearch-dashboards: Add `3.4.0` ([#1392]). - testing-tools: build testing tools subimages in workflow ([#1366]). - kafka: Add `4.1.1` ([#1395]). -- spark: Add `4.1.1`, use one Dockerfile per version and remove all HBase deps from the Spark4 image ([#1402]). +- spark: Add `4.1.1` ([#1402]). ### Changed @@ -27,6 +27,9 @@ All notable changes to this project will be documented in this file. - trino: Backport Kafka offset handling to 477 ([#1373]). - ubi: Bumped ubi9 and ubi10 hashes ([#1386]). - vector: Bumped from 0.49.0 to 0.52.0 ([#1387]). +- spark: Use one Dockerfile per major product version ([#1402]). + Remove all HBase dependencies from the Spark 4 image. + Pull logging dependencies with `mvn` instead of `curl` to remove manual maintenance in Nexus `packages`. ### Removed diff --git a/spark-k8s/Dockerfile.3 b/spark-k8s/Dockerfile.3 index 668242723..351dab6e9 100644 --- a/spark-k8s/Dockerfile.3 +++ b/spark-k8s/Dockerfile.3 @@ -239,16 +239,16 @@ WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/ext RUN < Date: Fri, 16 Jan 2026 16:06:37 +0100 Subject: [PATCH 06/12] update spark-connect-client versions --- CHANGELOG.md | 2 ++ spark-connect-client/boil-config.toml | 17 +++++------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c0021eca..69e08958e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file. - testing-tools: build testing tools subimages in workflow ([#1366]). - kafka: Add `4.1.1` ([#1395]). - spark: Add `4.1.1` ([#1402]). +- spark-connect-client: Add `4.1.1` ([#1402]). ### Changed @@ -38,6 +39,7 @@ All notable changes to this project will be documented in this file. - kafka: Remove `3.7.2` and `4.1.0` ([#1395]). - opa: remove 1.4.2 ([#1396]). - spark: Remove `3.5.6` and `4.0.1` ([#1402]). +- spark-connect-client: Remove `3.5.6` and `4.0.1` ([#1402]). ### Fixed diff --git a/spark-connect-client/boil-config.toml b/spark-connect-client/boil-config.toml index c0cfddf69..88d33dfe1 100644 --- a/spark-connect-client/boil-config.toml +++ b/spark-connect-client/boil-config.toml @@ -1,10 +1,3 @@ -[versions."3.5.6".local-images] -spark-k8s = "3.5.6" -java-base = "17" - -[versions."3.5.6".build-arguments] -python-version = "3.11" - [versions."3.5.7".local-images] spark-k8s = "3.5.7" java-base = "17" @@ -12,9 +5,9 @@ java-base = "17" [versions."3.5.7".build-arguments] python-version = "3.11" -[versions."4.0.1".local-images] -spark-k8s = "4.0.1" -java-base = "17" +[versions."4.1.1".local-images] +spark-k8s = "4.1.1" +java-base = "21" -[versions."4.0.1".build-arguments] -python-version = "3.11" +[versions."4.1.1".build-arguments] +python-version = "3.12" From a1446e41faba0ccf0d9ab5eda87d4d4547f39e69 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 16 Jan 2026 20:57:10 +0100 Subject: [PATCH 07/12] new spark-k8s/hbase-connectors image --- spark-k8s/Dockerfile.3 | 91 +------------- spark-k8s/boil-config.toml | 4 +- spark-k8s/hbase-connectors/Dockerfile | 119 ++++++++++++++++++ spark-k8s/hbase-connectors/boil-config.toml | 10 ++ .../0001-Fix-protobuf-on-aarch64.patch | 0 .../patches/{1.0.1 => 1.0.1_3}/patchable.toml | 0 6 files changed, 132 insertions(+), 92 deletions(-) create mode 100644 spark-k8s/hbase-connectors/Dockerfile create mode 100644 spark-k8s/hbase-connectors/boil-config.toml rename spark-k8s/hbase-connectors/stackable/patches/{1.0.1 => 1.0.1_3}/0001-Fix-protobuf-on-aarch64.patch (100%) rename spark-k8s/hbase-connectors/stackable/patches/{1.0.1 => 1.0.1_3}/patchable.toml (100%) diff --git a/spark-k8s/Dockerfile.3 b/spark-k8s/Dockerfile.3 index 351dab6e9..bf9be8cb4 100644 --- a/spark-k8s/Dockerfile.3 +++ b/spark-k8s/Dockerfile.3 @@ -33,96 +33,7 @@ EOF # hbase-connectors-builder: Build the Spark HBase connector and copy # required JARs into /stackable/spark/jars -FROM local-image/java-devel AS hbase-connectors-builder - -ARG PRODUCT_VERSION -ARG RELEASE_VERSION -ARG HADOOP_HADOOP_VERSION -# Reassign the arg to `HADOOP_VERSION` for better readability. -ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} -ARG HBASE_VERSION -ARG HBASE_CONNECTOR_VERSION -ARG STACKABLE_USER_UID - -WORKDIR /stackable - -# Copy the pom.xml file from the patched Spark source code to read the -# versions used by Spark. The pom.xml defines child modules which are -# not required and not copied, therefore mvn must be called with the -# parameter --non-recursive. -COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \ - /stackable/src/spark-k8s/patchable-work/worktree/${PRODUCT_VERSION}/pom.xml \ - spark/ - -# Patch the hbase-connectors source code -WORKDIR /stackable - -COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/patchable.toml /stackable/src/spark-k8s/hbase-connectors/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/${HBASE_CONNECTOR_VERSION} /stackable/src/spark-k8s/hbase-connectors/stackable/patches/${HBASE_CONNECTOR_VERSION} - -RUN <]' '{print $3}') - -# Get the Scala binary version used by Spark -SCALA_BINARY_VERSION=$(grep "scala.binary.version" /stackable/spark/pom.xml | head -n1 | awk -F '[<>]' '{print $3}') - -# Build the Spark HBase connector -# Skip the tests because the MiniHBaseCluster does not get ready for -# whatever reason: -# Caused by: java.lang.RuntimeException: Master not active after 30000ms -# at org.apache.hadoop.hbase.util.JVMClusterUtil.waitForEvent(JVMClusterUtil.java:221) -# at org.apache.hadoop.hbase.util.JVMClusterUtil.startup(JVMClusterUtil.java:177) -# at org.apache.hadoop.hbase.LocalHBaseCluster.startup(LocalHBaseCluster.java:407) -# at org.apache.hadoop.hbase.MiniHBaseCluster.init(MiniHBaseCluster.java:250) -mvn \ - --batch-mode \ - --no-transfer-progress \ - --define spark.version="${PRODUCT_VERSION}" \ - --define scala.version="${SCALA_VERSION}" \ - --define scala.binary.version="${SCALA_BINARY_VERSION}" \ - --define hadoop-three.version="${HADOOP_VERSION}" \ - --define hbase.version="${HBASE_VERSION}" \ - --define skipTests \ - --define maven.test.skip=true \ - clean package - -mkdir -p /stackable/spark/jars -ln -s "$(pwd)/hbase-spark/target/hbase-spark-${HBASE_CONNECTOR_VERSION}-stackable${RELEASE_VERSION}.jar" /stackable/spark/jars/hbase-spark-${HBASE_CONNECTOR_VERSION}-stackable${RELEASE_VERSION}.jar - -cd /stackable/spark/jars - -# Download log4j-slf4j-impl-x.x.x.jar containing the StaticLoggerBinder -# which is required by the connector. -# Spark contains only log4j-slf4j2-impl-x.x.x.jar but not -# log4j-slf4j-impl-x.x.x.jar. It is okay to have both JARs in the -# classpath as long as they have the same version. -mvn --non-recursive --file /stackable/spark/pom.xml \ - dependency:copy \ - -Dartifact=org.apache.logging.log4j:log4j-slf4j-impl:'${log4j.version}' \ - -DoutputDirectory=./jars -chmod g=u /stackable/hbase-connector-${HBASE_CONNECTOR_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz . -EOF - +FROM local-image/spark-k8s/hbase-connectors AS hbase-connectors-builder # spark-builder: Build Spark into /stackable/spark-${PRODUCT_VERSION}/dist, # download additional JARs and perform checks diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index d9a3336b5..9545f617e 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -6,6 +6,7 @@ containerfile = "Dockerfile.3" java-base = "17" java-devel = "17" hbase = "2.6.3" +"spark-k8s/hbase-connectors" = "1.0.1_3" [versions."3.5.7".build-arguments] python-version = "3.11" @@ -17,7 +18,7 @@ stax2-api-version = "4.2.1" # needs to match the jackson version h woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" -hbase-connector-version = "1.0.1" +hbase-connector-version = "1.0.1_3" [versions."4.1.1"] containerfile = "Dockerfile.4" @@ -38,4 +39,3 @@ stax2-api-version = "4.2.2" # needs to match the jackson version h woodstox-core-version = "7.1.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0/dependencies jmx-exporter-version = "1.3.0" tini-version = "0.19.0" -# hbase-connector-version = "1.0.1" # Not yet available for Spark 4.x https://github.com/apache/hbase-connectors/pull/130 diff --git a/spark-k8s/hbase-connectors/Dockerfile b/spark-k8s/hbase-connectors/Dockerfile new file mode 100644 index 000000000..f32ae2bcf --- /dev/null +++ b/spark-k8s/hbase-connectors/Dockerfile @@ -0,0 +1,119 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# check=error=true + +# The purpose of this stage is to gather jars and environment variables needed in the final stage. +# These are collected in the /stackable/spark directory. +FROM local-image/java-devel AS spark-source-builder + +ARG RELEASE_VERSION +ARG SPARK_VERSION +ARG STACKABLE_USER_UID + +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/patchable.toml /stackable/src/spark-k8s/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/${SPARK_VERSION} /stackable/src/spark-k8s/stackable/patches/${SPARK_VERSION} + +RUN <]' '{print $3}') + +# Get the Scala binary version used by Spark +SCALA_BINARY_VERSION=$(grep "scala.binary.version" pom.xml | head -n1 | awk -F '[<>]' '{print $3}') + +echo "SCALA_VERSION=${SCALA_VERSION}" > /stackable/spark/env +echo "SCALA_BINARY_VERSION=${SCALA_BINARY_VERSION}" >> /stackable/spark/env +echo "SPARK_VERSION=${SPARK_VERSION}" >> /stackable/spark/env +EOF + +# hbase-connectors-builder: Build the Spark HBase connector and copy +# required JARs into /stackable/spark/jars +FROM local-image/java-devel AS final + +ARG PRODUCT_VERSION +ARG RELEASE_VERSION +ARG HADOOP_VERSION +ARG HBASE_VERSION +ARG STACKABLE_USER_UID + +# Patch the hbase-connectors source code +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/patchable.toml /stackable/src/spark-k8s/hbase-connectors/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/${PRODUCT_VERSION} /stackable/src/spark-k8s/hbase-connectors/stackable/patches/${PRODUCT_VERSION} + +# Copy jars and env from spark-source-builder +COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \ + /stackable/spark/jars \ + spark/jars +COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \ + /stackable/spark/env \ + spark/env + +RUN < Date: Thu, 15 Jan 2026 13:06:37 +0100 Subject: [PATCH 08/12] chore(hive): Small cleanup after 4.2.0 version addition (#1397) * chore(hive): Small cleanup after 4.2.0 version addition * add to the template * add to the template --- .github/ISSUE_TEMPLATE/update-product-hive.md | 4 +++- hive/boil-config.toml | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/update-product-hive.md b/.github/ISSUE_TEMPLATE/update-product-hive.md index 2b1e39a94..de5af392a 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hive.md +++ b/.github/ISSUE_TEMPLATE/update-product-hive.md @@ -28,9 +28,11 @@ Add/Change/Remove anything that isn't applicable anymore ## Update tasks - [ ] Update `boil-config.toml` to reflect the agreed upon versions in the spreadsheet (including the removal of old versions). -- [ ] Upload new version (see `hive/upload_new_hive_version.sh`). - [ ] Update `boil-config.toml` to the latest supported version of JVM (base and devel). - [ ] Update other dependencies if applicable (eg: jmx_exporter, aws_java_sdk_bundle, etc). +- [ ] Initialize new product versions with patchable and add patches if applicable. +- [ ] Delete old patch directories. +- [ ] Check the corresponding operator (getting_started / kuttl / supported-versions) for usage of the versions. - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below. - [ ] Ensure prerequisites are up to date (required-external-components.adoc). - [ ] Update the version in demos. Add the PR(s) to the list below. diff --git a/hive/boil-config.toml b/hive/boil-config.toml index a216f20e7..aab12f907 100644 --- a/hive/boil-config.toml +++ b/hive/boil-config.toml @@ -70,9 +70,9 @@ java-devel = "21" [versions."4.2.0".build-arguments] jmx-exporter-version = "1.3.0" -# Keep consistent with the dependency from hadoop-aws: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.2 TODO: CHECK! +# Keep consistent with the dependency from hadoop-aws: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.2 aws-java-sdk-bundle-version = "2.29.52" -# Keep consistent with the dependency from hadoop-azure: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.2 TODO: CHECK! +# Keep consistent with the dependency from hadoop-azure: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.2 azure-storage-version = "7.0.1" -# Keep consistent with the dependency from azure-storage: https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 TODO: CHECK! +# Keep consistent with the dependency from azure-storage: https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 azure-keyvault-core-version = "1.0.0" From 5869472813f3b8652c0af286475af559cf15f680 Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:09:32 +0100 Subject: [PATCH 09/12] chore: Remove ZooKeeper 3.9.3 (#1401) * chore: Remove ZooKeeper 3.9.3 * chore: Update changelog --- CHANGELOG.md | 3 + zookeeper/boil-config.toml | 8 - .../3.9.3/0001-Add-CycloneDX-plugin.patch | 34 - ...ailure-to-reload-database-due-to-mis.patch | 64 -- ...etry-endlessly-to-establish-a-brand-.patch | 143 ---- ...ix-data-loss-due-to-propagation-of-d.patch | 611 ------------------ ...-jetty-version-to-fix-CVE-2024-13009.patch | 22 - ...-4.1.119.Final-to-fix-CVE-2025-24970.patch | 22 - .../stackable/patches/3.9.3/patchable.toml | 2 - 9 files changed, 3 insertions(+), 906 deletions(-) delete mode 100644 zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch delete mode 100644 zookeeper/stackable/patches/3.9.3/0002-ZOOKEEPER-4846-Failure-to-reload-database-due-to-mis.patch delete mode 100644 zookeeper/stackable/patches/3.9.3/0003-ZOOKEEPER-4921-Retry-endlessly-to-establish-a-brand-.patch delete mode 100644 zookeeper/stackable/patches/3.9.3/0004-ZOOKEEPER-4925-Fix-data-loss-due-to-propagation-of-d.patch delete mode 100644 zookeeper/stackable/patches/3.9.3/0005-Bumping-jetty-version-to-fix-CVE-2024-13009.patch delete mode 100644 zookeeper/stackable/patches/3.9.3/0006-Bumping-netty-to-4.1.119.Final-to-fix-CVE-2025-24970.patch delete mode 100644 zookeeper/stackable/patches/3.9.3/patchable.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 69e08958e..3073d2f88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ All notable changes to this project will be documented in this file. - opa: remove 1.4.2 ([#1396]). - spark: Remove `3.5.6` and `4.0.1` ([#1402]). - spark-connect-client: Remove `3.5.6` and `4.0.1` ([#1402]). +- opa: Remove `1.4.2` ([#1396]). +- zookeeper: Remove `3.9.3` ([#1401]). ### Fixed @@ -68,6 +70,7 @@ All notable changes to this project will be documented in this file. [#1395]: https://github.com/stackabletech/docker-images/pull/1395 [#1396]: https://github.com/stackabletech/docker-images/pull/1396 [#1402]: https://github.com/stackabletech/docker-images/pull/1402 +[#1401]: https://github.com/stackabletech/docker-images/pull/1401 ## [25.11.0] - 2025-11-07 diff --git a/zookeeper/boil-config.toml b/zookeeper/boil-config.toml index 80d3f24b4..cab00dcde 100644 --- a/zookeeper/boil-config.toml +++ b/zookeeper/boil-config.toml @@ -1,11 +1,3 @@ -[versions."3.9.3".local-images] -java-base = "17" -java-devel = "11" -"shared/logback" = "1.2.13" - -[versions."3.9.3".build-arguments] -jmx-exporter-version = "1.4.0" - [versions."3.9.4".local-images] java-base = "17" java-devel = "11" diff --git a/zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch b/zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch deleted file mode 100644 index 901191646..000000000 --- a/zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch +++ /dev/null @@ -1,34 +0,0 @@ -From f2dbb32161000b95032fbc6caee276f2c92552d8 Mon Sep 17 00:00:00 2001 -From: Lukas Voetmand -Date: Fri, 6 Sep 2024 17:53:52 +0200 -Subject: Add CycloneDX plugin - ---- - pom.xml | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/pom.xml b/pom.xml -index 6ef4011f..07ae7538 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -925,7 +925,7 @@ - - org.cyclonedx - cyclonedx-maven-plugin -- 2.7.9 -+ 2.8.0 - - - -@@ -1200,6 +1200,11 @@ - - org.cyclonedx - cyclonedx-maven-plugin -+ -+ application -+ 1.5 -+ false -+ - - - diff --git a/zookeeper/stackable/patches/3.9.3/0002-ZOOKEEPER-4846-Failure-to-reload-database-due-to-mis.patch b/zookeeper/stackable/patches/3.9.3/0002-ZOOKEEPER-4846-Failure-to-reload-database-due-to-mis.patch deleted file mode 100644 index 3890dabef..000000000 --- a/zookeeper/stackable/patches/3.9.3/0002-ZOOKEEPER-4846-Failure-to-reload-database-due-to-mis.patch +++ /dev/null @@ -1,64 +0,0 @@ -From 4004002a9ff08a539a94842ea12a2a449274e968 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Andor=20Moln=C3=A1r?= -Date: Tue, 11 Feb 2025 10:43:20 -0600 -Subject: ZOOKEEPER-4846: Failure to reload database due to missing ACL - -ZOOKEEPER-4846. Fix ACL reference on existing znode when trying to create -Reviewers: cnauroth, eolivelli, ztzg -Author: anmolnar -Closes #2222 from anmolnar/ZOOKEEPER-4846 ---- - .../org/apache/zookeeper/server/DataTree.java | 5 +++-- - .../apache/zookeeper/server/DataTreeTest.java | 16 ++++++++++++++++ - 2 files changed, 19 insertions(+), 2 deletions(-) - -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java -index 3b61c80d..af937f83 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java -@@ -462,8 +462,9 @@ public class DataTree { - // we did for the global sessions. - Long acls = aclCache.convertAcls(acl); - -- Set children = parent.getChildren(); -- if (children.contains(childName)) { -+ DataNode existingChild = nodes.get(path); -+ if (existingChild != null) { -+ existingChild.acl = acls; - throw new NodeExistsException(); - } - -diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java -index 07a69f14..fc20ed32 100644 ---- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java -+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java -@@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; - import static org.junit.jupiter.api.Assertions.assertNotEquals; - import static org.junit.jupiter.api.Assertions.assertNotNull; - import static org.junit.jupiter.api.Assertions.assertNull; -+import static org.junit.jupiter.api.Assertions.assertThrows; - import static org.junit.jupiter.api.Assertions.assertTrue; - import java.io.ByteArrayInputStream; - import java.io.ByteArrayOutputStream; -@@ -631,6 +632,21 @@ public class DataTreeTest extends ZKTestCase { - } - } - -+ @Test -+ public void testCreateNodeFixMissingACL() throws Exception { -+ DataTree dt = new DataTree(); -+ ReferenceCountedACLCache aclCache = dt.getReferenceCountedAclCache(); -+ -+ dt.createNode("/the_parent", new byte[0], ZooDefs.Ids.CREATOR_ALL_ACL, -1, 1, 1, 0); -+ Long aclId = dt.getNode("/the_parent").acl; -+ aclCache.removeUsage(aclId); -+ aclCache.purgeUnused(); -+ // try to re-create the parent -> throws NodeExistsException, but fixes the deleted ACL -+ assertThrows(NodeExistsException.class, () -> -+ dt.createNode("/the_parent", new byte[0], ZooDefs.Ids.CREATOR_ALL_ACL, -1, 1, 1, 0)); -+ dt.createNode("/the_parent/the_child", new byte[0], ZooDefs.Ids.CREATOR_ALL_ACL, -1, 2, 2, 2); -+ } -+ - private DataTree buildDataTreeForTest() { - final DataTree dt = new DataTree(); - assertEquals(dt.lastProcessedZxid, 0); diff --git a/zookeeper/stackable/patches/3.9.3/0003-ZOOKEEPER-4921-Retry-endlessly-to-establish-a-brand-.patch b/zookeeper/stackable/patches/3.9.3/0003-ZOOKEEPER-4921-Retry-endlessly-to-establish-a-brand-.patch deleted file mode 100644 index 734dc2479..000000000 --- a/zookeeper/stackable/patches/3.9.3/0003-ZOOKEEPER-4921-Retry-endlessly-to-establish-a-brand-.patch +++ /dev/null @@ -1,143 +0,0 @@ -From 90e8e0f44e8a884765b6e7afe8bd779d59136fad Mon Sep 17 00:00:00 2001 -From: Kezhu Wang -Date: Sat, 26 Apr 2025 12:04:01 +0800 -Subject: ZOOKEEPER-4921: Retry endlessly to establish a brand-new session - -This partially rollback ZOOKEEPER-4508 to keep consistent with versions -prior to 3.9.3 (excluded), so to maintain compatibility with third party -libraries. - -Refs: ZOOKEEPER-4508, ZOOKEEPER-4921, ZOOKEEPER-4923 and -https://lists.apache.org/thread/nfb9z7rhgglbjzfxvg4z2m3pks53b3c1 ---- - .../java/org/apache/zookeeper/ClientCnxn.java | 2 +- - .../zookeeper/test/SessionTimeoutTest.java | 65 +++++++++++++------ - 2 files changed, 47 insertions(+), 20 deletions(-) - -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java -index 0bf616c6..207bb8c4 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java -@@ -1242,7 +1242,7 @@ public class ClientCnxn { - to = connectTimeout - clientCnxnSocket.getIdleSend(); - } - -- int expiration = expirationTimeout - clientCnxnSocket.getIdleRecv(); -+ int expiration = sessionId == 0 ? Integer.MAX_VALUE : expirationTimeout - clientCnxnSocket.getIdleRecv(); - if (expiration <= 0) { - String warnInfo = String.format( - "Client session timed out, have not heard from server in %dms for session id 0x%s", -diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java -index 7a59f5eb..9f5943f6 100644 ---- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java -+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java -@@ -18,6 +18,9 @@ - - package org.apache.zookeeper.test; - -+import static org.hamcrest.MatcherAssert.assertThat; -+import static org.hamcrest.Matchers.greaterThanOrEqualTo; -+import static org.hamcrest.Matchers.lessThan; - import static org.junit.jupiter.api.Assertions.assertNotNull; - import static org.junit.jupiter.api.Assertions.assertNull; - import static org.junit.jupiter.api.Assertions.assertThrows; -@@ -31,12 +34,15 @@ import java.util.List; - import java.util.concurrent.CompletableFuture; - import java.util.concurrent.CountDownLatch; - import java.util.concurrent.TimeUnit; -+import java.util.concurrent.TimeoutException; - import org.apache.zookeeper.CreateMode; - import org.apache.zookeeper.KeeperException; - import org.apache.zookeeper.TestableZooKeeper; - import org.apache.zookeeper.WatchedEvent; - import org.apache.zookeeper.Watcher; - import org.apache.zookeeper.ZooDefs; -+import org.apache.zookeeper.ZooKeeper; -+import org.apache.zookeeper.common.Time; - import org.junit.jupiter.api.BeforeEach; - import org.junit.jupiter.api.Test; - import org.slf4j.Logger; -@@ -54,6 +60,21 @@ public class SessionTimeoutTest extends ClientBase { - zk = createClient(); - } - -+ private static class ExpiredWatcher implements Watcher { -+ public volatile CompletableFuture expired = new CompletableFuture<>(); -+ -+ synchronized void reset() { -+ expired = new CompletableFuture<>(); -+ } -+ -+ @Override -+ public synchronized void process(WatchedEvent event) { -+ if (event.getState() == Event.KeeperState.Expired) { -+ expired.complete(null); -+ } -+ } -+ } -+ - private static class BusyServer implements AutoCloseable { - private final ServerSocket server; - private final Socket client; -@@ -143,17 +164,24 @@ public class SessionTimeoutTest extends ClientBase { - // stop client also to gain less distraction - zk.close(); - -- // small connection timeout to gain quick ci feedback -- int sessionTimeout = 3000; -- CompletableFuture expired = new CompletableFuture<>(); -+ // given: established session -+ int sessionTimeout = 3000; // small connection timeout to gain quick ci feedback -+ ExpiredWatcher watcher = new ExpiredWatcher(); - zk = createClient(new CountdownWatcher(), hostPort, sessionTimeout); -- zk.register(event -> { -- if (event.getState() == Watcher.Event.KeeperState.Expired) { -- expired.complete(null); -- } -- }); -+ zk.register(watcher); -+ -+ // when: all server down -+ long start = Time.currentElapsedTime(); -+ zk.sync("/"); // touch timeout counts - stopServer(); -- expired.join(); -+ -+ // then: get Expired after session timeout -+ watcher.expired.join(); -+ long elapsed = Time.currentElapsedTime() - start; -+ assertThat(elapsed, greaterThanOrEqualTo((long) zk.getSessionTimeout())); -+ assertThat(elapsed, lessThan(zk.getSessionTimeout() * 10L)); -+ -+ // then: future request will get SessionExpiredException - assertThrows(KeeperException.SessionExpiredException.class, () -> zk.exists("/", null)); - } - -@@ -162,18 +190,17 @@ public class SessionTimeoutTest extends ClientBase { - // stop client also to gain less distraction - zk.close(); - -+ // given: unavailable cluster - stopServer(); - -- // small connection timeout to gain quick ci feedback -- int sessionTimeout = 3000; -- CompletableFuture expired = new CompletableFuture<>(); -- new TestableZooKeeper(hostPort, sessionTimeout, event -> { -- if (event.getState() == Watcher.Event.KeeperState.Expired) { -- expired.complete(null); -- } -- }); -- expired.join(); -- assertThrows(KeeperException.SessionExpiredException.class, () -> zk.exists("/", null)); -+ // when: try to establish a brand-new session -+ int sessionTimeout = 300; // small connection timeout to gain quick ci feedback -+ ExpiredWatcher watcher = new ExpiredWatcher(); -+ try (ZooKeeper zk = new ZooKeeper(hostPort, sessionTimeout, watcher)) { -+ // then: never Expired -+ assertThrows(TimeoutException.class, () -> watcher.expired.get(3 * sessionTimeout, TimeUnit.MILLISECONDS)); -+ assertThrows(KeeperException.ConnectionLossException.class, () -> zk.exists("/", null)); -+ } - } - - @Test diff --git a/zookeeper/stackable/patches/3.9.3/0004-ZOOKEEPER-4925-Fix-data-loss-due-to-propagation-of-d.patch b/zookeeper/stackable/patches/3.9.3/0004-ZOOKEEPER-4925-Fix-data-loss-due-to-propagation-of-d.patch deleted file mode 100644 index 469ffd8d3..000000000 --- a/zookeeper/stackable/patches/3.9.3/0004-ZOOKEEPER-4925-Fix-data-loss-due-to-propagation-of-d.patch +++ /dev/null @@ -1,611 +0,0 @@ -From d16264bd13ce61ae5a4375e30e9d1787c1206747 Mon Sep 17 00:00:00 2001 -From: Kezhu Wang -Date: Wed, 30 Apr 2025 11:45:05 +0800 -Subject: ZOOKEEPER-4925: Fix data loss due to propagation of discontinuous - committedLog - -There are two variants of `ZooKeeperServer::processTxn`. Those two -variants diverge significantly since ZOOKEEPER-3484. -`processTxn(Request request)` pops outstanding change from -`outstandingChanges` and adds txn to `committedLog` for follower to sync -in addition to what `processTxn(TxnHeader hdr, Record txn)` does. The -`Learner` uses `processTxn(TxnHeader hdr, Record txn)` to commit txn to -memory after ZOOKEEPER-4394, which means it leaves `committedLog` -untouched in `SYNCHRONIZATION` phase. - -This way, a stale follower will have hole in its `committedLog` after -joining cluster. The stale follower will propagate the in memory hole -to other stale nodes after becoming leader. This causes data loss. - -The test case fails on master and 3.9.3, and passes on 3.9.2. So only -3.9.3 is affected. - -This commit drops `processTxn(TxnHeader hdr, Record txn)` as -`processTxn(Request request)` is capable in `SYNCHRONIZATION` phase too. - -Also, this commit rejects discontinuous proposals in `syncWithLeader` -and `committedLog`, so to avoid possible data loss. - -Refs: ZOOKEEPER-4925, ZOOKEEPER-4394, ZOOKEEPER-3484 - -Add separated code to enforce continuous proposals ---- - .../org/apache/zookeeper/server/Request.java | 13 +++ - .../apache/zookeeper/server/TxnLogEntry.java | 4 + - .../apache/zookeeper/server/ZKDatabase.java | 28 +++-- - .../zookeeper/server/ZooKeeperServer.java | 21 ++-- - .../zookeeper/server/quorum/Follower.java | 4 +- - .../quorum/FollowerZooKeeperServer.java | 34 ++---- - .../zookeeper/server/quorum/Learner.java | 58 ++++++---- - .../zookeeper/server/quorum/Observer.java | 11 +- - .../zookeeper/server/TxnLogDigestTest.java | 2 + - .../zookeeper/server/ZxidRolloverTest.java | 2 + - .../server/quorum/QuorumSyncTest.java | 100 ++++++++++++++++++ - 11 files changed, 196 insertions(+), 81 deletions(-) - create mode 100644 zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumSyncTest.java - -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/Request.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/Request.java -index c174fdd1..ad507137 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/Request.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/Request.java -@@ -78,6 +78,19 @@ public class Request { - this.authInfo = null; - } - -+ public Request(TxnHeader hdr, Record txn, TxnDigest digest) { -+ this.sessionId = hdr.getClientId(); -+ this.cxid = hdr.getCxid(); -+ this.type = hdr.getType(); -+ this.hdr = hdr; -+ this.txn = txn; -+ this.zxid = hdr.getZxid(); -+ this.request = null; -+ this.cnxn = null; -+ this.authInfo = null; -+ this.txnDigest = digest; -+ } -+ - public final long sessionId; - - public final int cxid; -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/TxnLogEntry.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/TxnLogEntry.java -index 352eb81d..409fd21f 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/TxnLogEntry.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/TxnLogEntry.java -@@ -47,4 +47,8 @@ public final class TxnLogEntry { - public TxnDigest getDigest() { - return digest; - } -+ -+ public Request toRequest() { -+ return new Request(header, txn, digest); -+ } - } -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java -index 7258daa7..7a26d836 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java -@@ -58,6 +58,7 @@ import org.apache.zookeeper.server.quorum.Leader.Proposal; - import org.apache.zookeeper.server.quorum.Leader.PureRequestProposal; - import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; - import org.apache.zookeeper.server.util.SerializeUtils; -+import org.apache.zookeeper.server.util.ZxidUtils; - import org.apache.zookeeper.txn.TxnDigest; - import org.apache.zookeeper.txn.TxnHeader; - import org.slf4j.Logger; -@@ -82,6 +83,8 @@ public class ZKDatabase { - protected FileTxnSnapLog snapLog; - protected long minCommittedLog, maxCommittedLog; - -+ private final boolean allowDiscontinuousProposals = Boolean.getBoolean("zookeeper.test.allowDiscontinuousProposals"); -+ - /** - * Default value is to use snapshot if txnlog size exceeds 1/3 the size of snapshot - */ -@@ -170,8 +173,6 @@ public class ZKDatabase { - * data structures in zkdatabase. - */ - public void clear() { -- minCommittedLog = 0; -- maxCommittedLog = 0; - /* to be safe we just create a new - * datatree. - */ -@@ -182,6 +183,8 @@ public class ZKDatabase { - try { - lock.lock(); - committedLog.clear(); -+ minCommittedLog = 0; -+ maxCommittedLog = 0; - } finally { - lock.unlock(); - } -@@ -320,17 +323,30 @@ public class ZKDatabase { - WriteLock wl = logLock.writeLock(); - try { - wl.lock(); -- if (committedLog.size() > commitLogCount) { -- committedLog.remove(); -- minCommittedLog = committedLog.peek().getZxid(); -- } - if (committedLog.isEmpty()) { - minCommittedLog = request.zxid; - maxCommittedLog = request.zxid; -+ } else if (request.zxid <= maxCommittedLog) { -+ // This could happen if lastProcessedZxid is rewinded and database is re-synced. -+ // Currently, it only happens in test codes, but it should also be safe for production path. -+ return; -+ } else if (!allowDiscontinuousProposals -+ && request.zxid != maxCommittedLog + 1 -+ && ZxidUtils.getEpochFromZxid(request.zxid) <= ZxidUtils.getEpochFromZxid(maxCommittedLog)) { -+ String msg = String.format( -+ "Committed proposal cached out of order: 0x%s is not the next proposal of 0x%s", -+ ZxidUtils.zxidToString(request.zxid), -+ ZxidUtils.zxidToString(maxCommittedLog)); -+ LOG.error(msg); -+ throw new IllegalStateException(msg); - } - PureRequestProposal p = new PureRequestProposal(request); - committedLog.add(p); - maxCommittedLog = p.getZxid(); -+ if (committedLog.size() > commitLogCount) { -+ committedLog.remove(); -+ minCommittedLog = committedLog.peek().getZxid(); -+ } - } finally { - wl.unlock(); - } -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java -index 6740f6d5..14dd59b8 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java -@@ -1846,13 +1846,6 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { - cnxn.sendResponse(replyHeader, record, "response"); - } - -- // entry point for quorum/Learner.java -- public ProcessTxnResult processTxn(TxnHeader hdr, Record txn) { -- processTxnForSessionEvents(null, hdr, txn); -- return processTxnInDB(hdr, txn, null); -- } -- -- // entry point for FinalRequestProcessor.java - public ProcessTxnResult processTxn(Request request) { - TxnHeader hdr = request.getHdr(); - processTxnForSessionEvents(request, hdr, request.getTxn()); -@@ -1864,8 +1857,10 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { - if (!writeRequest && !quorumRequest) { - return new ProcessTxnResult(); - } -+ -+ ProcessTxnResult rc; - synchronized (outstandingChanges) { -- ProcessTxnResult rc = processTxnInDB(hdr, request.getTxn(), request.getTxnDigest()); -+ rc = processTxnInDB(hdr, request.getTxn(), request.getTxnDigest()); - - // request.hdr is set for write requests, which are the only ones - // that add to outstandingChanges. -@@ -1886,13 +1881,13 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { - } - } - } -+ } - -- // do not add non quorum packets to the queue. -- if (quorumRequest) { -- getZKDatabase().addCommittedProposal(request); -- } -- return rc; -+ // do not add non quorum packets to the queue. -+ if (quorumRequest) { -+ getZKDatabase().addCommittedProposal(request); - } -+ return rc; - } - - private void processTxnForSessionEvents(Request request, TxnHeader hdr, Record txn) { -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java -index 0eff9d24..ca99974c 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java -@@ -35,7 +35,6 @@ import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; - import org.apache.zookeeper.server.util.SerializeUtils; - import org.apache.zookeeper.server.util.ZxidUtils; - import org.apache.zookeeper.txn.SetDataTxn; --import org.apache.zookeeper.txn.TxnDigest; - import org.apache.zookeeper.txn.TxnHeader; - - /** -@@ -164,7 +163,6 @@ public class Follower extends Learner { - TxnLogEntry logEntry = SerializeUtils.deserializeTxn(qp.getData()); - TxnHeader hdr = logEntry.getHeader(); - Record txn = logEntry.getTxn(); -- TxnDigest digest = logEntry.getDigest(); - if (hdr.getZxid() != lastQueued + 1) { - LOG.warn( - "Got zxid 0x{} expected 0x{}", -@@ -179,7 +177,7 @@ public class Follower extends Learner { - self.setLastSeenQuorumVerifier(qv, true); - } - -- fzk.logRequest(hdr, txn, digest); -+ fzk.logRequest(logEntry.toRequest()); - if (hdr != null) { - /* - * Request header is created only by the leader, so this is only set -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java -index b6766199..1b0b5cd9 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java -@@ -22,7 +22,6 @@ import java.io.IOException; - import java.util.concurrent.ConcurrentLinkedQueue; - import java.util.concurrent.LinkedBlockingQueue; - import javax.management.JMException; --import org.apache.jute.Record; - import org.apache.zookeeper.jmx.MBeanRegistry; - import org.apache.zookeeper.metrics.MetricsContext; - import org.apache.zookeeper.server.ExitCode; -@@ -33,8 +32,6 @@ import org.apache.zookeeper.server.ServerMetrics; - import org.apache.zookeeper.server.SyncRequestProcessor; - import org.apache.zookeeper.server.ZKDatabase; - import org.apache.zookeeper.server.persistence.FileTxnSnapLog; --import org.apache.zookeeper.txn.TxnDigest; --import org.apache.zookeeper.txn.TxnHeader; - import org.apache.zookeeper.util.ServiceUtils; - import org.slf4j.Logger; - import org.slf4j.LoggerFactory; -@@ -79,20 +76,17 @@ public class FollowerZooKeeperServer extends LearnerZooKeeperServer { - - LinkedBlockingQueue pendingTxns = new LinkedBlockingQueue<>(); - -- public void logRequest(TxnHeader hdr, Record txn, TxnDigest digest) { -- final Request request = buildRequestToProcess(hdr, txn, digest); -+ public void logRequest(Request request) { -+ if ((request.zxid & 0xffffffffL) != 0) { -+ pendingTxns.add(request); -+ } - syncProcessor.processRequest(request); - } - - /** -- * Build a request for the txn and append it to the transaction log -- * @param hdr the txn header -- * @param txn the txn -- * @param digest the digest of txn -+ * Append txn request to the transaction log directly without go through request processors. - */ -- public void appendRequest(final TxnHeader hdr, final Record txn, final TxnDigest digest) throws IOException { -- final Request request = new Request(hdr.getClientId(), hdr.getCxid(), hdr.getType(), hdr, txn, hdr.getZxid()); -- request.setTxnDigest(digest); -+ public void appendRequest(Request request) throws IOException { - getZKDatabase().append(request); - } - -@@ -188,20 +182,4 @@ public class FollowerZooKeeperServer extends LearnerZooKeeperServer { - rootContext.unregisterGauge("synced_observers"); - - } -- -- /** -- * Build a request for the txn -- * @param hdr the txn header -- * @param txn the txn -- * @param digest the digest of txn -- * @return a request moving through a chain of RequestProcessors -- */ -- private Request buildRequestToProcess(final TxnHeader hdr, final Record txn, final TxnDigest digest) { -- final Request request = new Request(hdr.getClientId(), hdr.getCxid(), hdr.getType(), hdr, txn, hdr.getZxid()); -- request.setTxnDigest(digest); -- if ((request.zxid & 0xffffffffL) != 0) { -- pendingTxns.add(request); -- } -- return request; -- } - } -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java -index 1ef99e50..adf0ef6e 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java -@@ -82,6 +82,10 @@ public class Learner { - Record rec; - TxnDigest digest; - -+ Request toRequest() { -+ return new Request(hdr, rec, digest); -+ } -+ - } - - QuorumPeer self; -@@ -535,6 +539,27 @@ public class Learner { - } - } - -+ long enforceContinuousProposal(long lastQueued, PacketInFlight pif) throws Exception { -+ if (lastQueued == 0) { -+ LOG.info("DIFF sync got first proposal 0x{}", Long.toHexString(pif.hdr.getZxid())); -+ } else if (pif.hdr.getZxid() != lastQueued + 1) { -+ if (ZxidUtils.getEpochFromZxid(pif.hdr.getZxid()) <= ZxidUtils.getEpochFromZxid(lastQueued)) { -+ String msg = String.format( -+ "DIFF sync got proposal 0x%s, last queued 0x%s, expected 0x%s", -+ Long.toHexString(pif.hdr.getZxid()), Long.toHexString(lastQueued), -+ Long.toHexString(lastQueued + 1)); -+ LOG.error(msg); -+ throw new Exception(msg); -+ } -+ // We can't tell whether it is a data loss. Given that new epoch is rare, -+ // log at warn should not be too verbose. -+ LOG.warn("DIFF sync got new epoch proposal 0x{}, last queued 0x{}, expected 0x{}", -+ Long.toHexString(pif.hdr.getZxid()), Long.toHexString(lastQueued), -+ Long.toHexString(lastQueued + 1)); -+ } -+ return pif.hdr.getZxid(); -+ } -+ - /** - * Finally, synchronize our history with the Leader (if Follower) - * or the LearnerMaster (if Observer). -@@ -609,6 +634,8 @@ public class Learner { - zk.getZKDatabase().initConfigInZKDatabase(self.getQuorumVerifier()); - zk.createSessionTracker(); - -+ // TODO: Ideally, this should be lastProcessZxid(a.k.a. QuorumPacket::zxid from above), but currently -+ // LearnerHandler does not guarantee this. So, let's be conservative and keep it unchange for now. - long lastQueued = 0; - - // in Zab V1.0 (ZK 3.4+) we might take a snapshot when we get the NEWLEADER message, but in pre V1.0 -@@ -630,13 +657,7 @@ public class Learner { - pif.hdr = logEntry.getHeader(); - pif.rec = logEntry.getTxn(); - pif.digest = logEntry.getDigest(); -- if (pif.hdr.getZxid() != lastQueued + 1) { -- LOG.warn( -- "Got zxid 0x{} expected 0x{}", -- Long.toHexString(pif.hdr.getZxid()), -- Long.toHexString(lastQueued + 1)); -- } -- lastQueued = pif.hdr.getZxid(); -+ lastQueued = enforceContinuousProposal(lastQueued, pif); - - if (pif.hdr.getType() == OpCode.reconfig) { - SetDataTxn setDataTxn = (SetDataTxn) pif.rec; -@@ -666,7 +687,7 @@ public class Learner { - Long.toHexString(qp.getZxid()), - Long.toHexString(pif.hdr.getZxid())); - } else { -- zk.processTxn(pif.hdr, pif.rec); -+ zk.processTxn(pif.toRequest()); - packetsNotLogged.remove(); - } - } else { -@@ -696,18 +717,11 @@ public class Learner { - packet.rec = logEntry.getTxn(); - packet.hdr = logEntry.getHeader(); - packet.digest = logEntry.getDigest(); -- // Log warning message if txn comes out-of-order -- if (packet.hdr.getZxid() != lastQueued + 1) { -- LOG.warn( -- "Got zxid 0x{} expected 0x{}", -- Long.toHexString(packet.hdr.getZxid()), -- Long.toHexString(lastQueued + 1)); -- } -- lastQueued = packet.hdr.getZxid(); -+ lastQueued = enforceContinuousProposal(lastQueued, packet); - } - if (!writeToTxnLog) { - // Apply to db directly if we haven't taken the snapshot -- zk.processTxn(packet.hdr, packet.rec); -+ zk.processTxn(packet.toRequest()); - } else { - packetsNotLogged.add(packet); - packetsCommitted.add(qp.getZxid()); -@@ -780,8 +794,9 @@ public class Learner { - continue; - } - packetsNotLogged.removeFirst(); -- fzk.appendRequest(pif.hdr, pif.rec, pif.digest); -- fzk.processTxn(pif.hdr, pif.rec); -+ Request request = pif.toRequest(); -+ fzk.appendRequest(request); -+ fzk.processTxn(request); - } - - // @see https://issues.apache.org/jira/browse/ZOOKEEPER-4646 -@@ -823,7 +838,7 @@ public class Learner { - if (zk instanceof FollowerZooKeeperServer) { - FollowerZooKeeperServer fzk = (FollowerZooKeeperServer) zk; - for (PacketInFlight p : packetsNotLogged) { -- fzk.logRequest(p.hdr, p.rec, p.digest); -+ fzk.logRequest(p.toRequest()); - } - LOG.info("{} txns have been logged asynchronously", packetsNotLogged.size()); - -@@ -847,8 +862,7 @@ public class Learner { - continue; - } - packetsCommitted.remove(); -- Request request = new Request(p.hdr.getClientId(), p.hdr.getCxid(), p.hdr.getType(), p.hdr, p.rec, -1); -- request.setTxnDigest(p.digest); -+ Request request = p.toRequest(); - ozk.commitRequest(request); - } - } else { -diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java -index d3aa41b5..334fa54c 100644 ---- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java -+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java -@@ -202,12 +202,8 @@ public class Observer extends Learner { - case Leader.INFORM: - ServerMetrics.getMetrics().LEARNER_COMMIT_RECEIVED_COUNT.add(1); - logEntry = SerializeUtils.deserializeTxn(qp.getData()); -- hdr = logEntry.getHeader(); -- txn = logEntry.getTxn(); -- digest = logEntry.getDigest(); -- Request request = new Request(hdr.getClientId(), hdr.getCxid(), hdr.getType(), hdr, txn, 0); -+ Request request = logEntry.toRequest(); - request.logLatency(ServerMetrics.getMetrics().COMMIT_PROPAGATION_LATENCY); -- request.setTxnDigest(digest); - ObserverZooKeeperServer obs = (ObserverZooKeeperServer) zk; - obs.commitRequest(request); - break; -@@ -219,13 +215,10 @@ public class Observer extends Learner { - byte[] remainingdata = new byte[buffer.remaining()]; - buffer.get(remainingdata); - logEntry = SerializeUtils.deserializeTxn(remainingdata); -- hdr = logEntry.getHeader(); - txn = logEntry.getTxn(); -- digest = logEntry.getDigest(); - QuorumVerifier qv = self.configFromString(new String(((SetDataTxn) txn).getData(), UTF_8)); - -- request = new Request(hdr.getClientId(), hdr.getCxid(), hdr.getType(), hdr, txn, 0); -- request.setTxnDigest(digest); -+ request = logEntry.toRequest(); - obs = (ObserverZooKeeperServer) zk; - - boolean majorChange = self.processReconfig(qv, suggestedLeaderId, qp.getZxid(), true); -diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/TxnLogDigestTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/TxnLogDigestTest.java -index 75d6fe68..b52ea341 100644 ---- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/TxnLogDigestTest.java -+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/TxnLogDigestTest.java -@@ -60,6 +60,7 @@ public class TxnLogDigestTest extends ClientBase { - - @BeforeEach - public void setUp() throws Exception { -+ System.setProperty("zookeeper.test.allowDiscontinuousProposals", "true"); - super.setUp(); - server = serverFactory.getZooKeeperServer(); - zk = createClient(); -@@ -67,6 +68,7 @@ public class TxnLogDigestTest extends ClientBase { - - @AfterEach - public void tearDown() throws Exception { -+ System.clearProperty("zookeeper.test.allowDiscontinuousProposals"); - // server will be closed in super.tearDown - super.tearDown(); - -diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java -index 031ccc2f..b23fd80a 100644 ---- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java -+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZxidRolloverTest.java -@@ -60,6 +60,7 @@ public class ZxidRolloverTest extends ZKTestCase { - @BeforeEach - public void setUp() throws Exception { - System.setProperty("zookeeper.admin.enableServer", "false"); -+ System.setProperty("zookeeper.test.allowDiscontinuousProposals", "true"); - - // set the snap count to something low so that we force log rollover - // and verify that is working as part of the epoch rollover. -@@ -215,6 +216,7 @@ public class ZxidRolloverTest extends ZKTestCase { - - @AfterEach - public void tearDown() throws Exception { -+ System.clearProperty("zookeeper.test.allowDiscontinuousProposals"); - LOG.info("tearDown starting"); - for (int i = 0; i < zkClients.length; i++) { - zkClients[i].close(); -diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumSyncTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumSyncTest.java -new file mode 100644 -index 00000000..c4b7720c ---- /dev/null -+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumSyncTest.java -@@ -0,0 +1,100 @@ -+/* -+ * Licensed to the Apache Software Foundation (ASF) under one -+ * or more contributor license agreements. See the NOTICE file -+ * distributed with this work for additional information -+ * regarding copyright ownership. The ASF licenses this file -+ * to you under the Apache License, Version 2.0 (the -+ * "License"); you may not use this file except in compliance -+ * with the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.zookeeper.server.quorum; -+ -+import static org.junit.jupiter.api.Assertions.assertNotNull; -+import java.util.Comparator; -+import org.apache.zookeeper.CreateMode; -+import org.apache.zookeeper.ZKTestCase; -+import org.apache.zookeeper.ZooDefs; -+import org.apache.zookeeper.ZooKeeper; -+import org.apache.zookeeper.test.ClientBase; -+import org.apache.zookeeper.test.QuorumUtil; -+import org.junit.jupiter.api.AfterEach; -+import org.junit.jupiter.api.Test; -+ -+public class QuorumSyncTest extends ZKTestCase { -+ private QuorumUtil qu; -+ -+ @AfterEach -+ public void tearDown() throws Exception { -+ if (qu != null) { -+ qu.shutdownAll(); -+ } -+ } -+ -+ @Test -+ public void testStaleDiffSync() throws Exception { -+ qu = new QuorumUtil(2); -+ qu.startAll(); -+ -+ int[] followerIds = qu.getFollowerQuorumPeers() -+ .stream() -+ .sorted(Comparator.comparingLong(QuorumPeer::getMyId).reversed()) -+ .mapToInt(peer -> (int) peer.getMyId()).toArray(); -+ -+ int follower1 = followerIds[0]; -+ int follower2 = followerIds[1]; -+ -+ String leaderConnectString = qu.getConnectString(qu.getLeaderQuorumPeer()); -+ try (ZooKeeper zk = ClientBase.createZKClient(leaderConnectString)) { -+ qu.shutdown(follower2); -+ -+ for (int i = 0; i < 10; i++) { -+ zk.create("/foo" + i, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); -+ } -+ -+ qu.shutdown(follower1); -+ -+ for (int i = 0; i < 10; i++) { -+ zk.create("/bar" + i, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); -+ } -+ -+ qu.restart(follower1); -+ } -+ -+ try (ZooKeeper zk = ClientBase.createZKClient(qu.getConnectionStringForServer(follower1))) { -+ for (int i = 0; i < 10; i++) { -+ String path = "/foo" + i; -+ assertNotNull(zk.exists(path, false), path + " not found"); -+ } -+ -+ for (int i = 0; i < 10; i++) { -+ String path = "/bar" + i; -+ assertNotNull(zk.exists(path, false), path + " not found"); -+ } -+ } -+ -+ qu.shutdown(qu.getLeaderServer()); -+ -+ qu.restart(follower2); -+ -+ try (ZooKeeper zk = ClientBase.createZKClient(qu.getConnectionStringForServer(follower2))) { -+ for (int i = 0; i < 10; i++) { -+ String path = "/foo" + i; -+ assertNotNull(zk.exists(path, false), path + " not found"); -+ } -+ -+ for (int i = 0; i < 10; i++) { -+ String path = "/bar" + i; -+ assertNotNull(zk.exists(path, false), path + " not found"); -+ } -+ } -+ } -+} diff --git a/zookeeper/stackable/patches/3.9.3/0005-Bumping-jetty-version-to-fix-CVE-2024-13009.patch b/zookeeper/stackable/patches/3.9.3/0005-Bumping-jetty-version-to-fix-CVE-2024-13009.patch deleted file mode 100644 index a5ace456e..000000000 --- a/zookeeper/stackable/patches/3.9.3/0005-Bumping-jetty-version-to-fix-CVE-2024-13009.patch +++ /dev/null @@ -1,22 +0,0 @@ -From d5ec0e10f1e2c967cd1bbc9aaeacc4f83705f1bf Mon Sep 17 00:00:00 2001 -From: Maxi Wittich -Date: Tue, 17 Jun 2025 15:39:44 +0200 -Subject: Bumping jetty version to fix CVE-2024-13009 - ---- - pom.xml | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/pom.xml b/pom.xml -index 07ae7538..9c201245 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -560,7 +560,7 @@ - 2.2 - 1.5.0 - 4.1.113.Final -- 9.4.56.v20240826 -+ 9.4.57.v20241219 - 2.15.2 - 2.14.6 - 1.1.10.5 diff --git a/zookeeper/stackable/patches/3.9.3/0006-Bumping-netty-to-4.1.119.Final-to-fix-CVE-2025-24970.patch b/zookeeper/stackable/patches/3.9.3/0006-Bumping-netty-to-4.1.119.Final-to-fix-CVE-2025-24970.patch deleted file mode 100644 index 1cebf4686..000000000 --- a/zookeeper/stackable/patches/3.9.3/0006-Bumping-netty-to-4.1.119.Final-to-fix-CVE-2025-24970.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 60f6980c40d9bdc3b9a447d68fd9c4c02da7d3de Mon Sep 17 00:00:00 2001 -From: Maxi Wittich -Date: Tue, 17 Jun 2025 16:53:38 +0200 -Subject: Bumping netty to 4.1.119.Final to fix CVE-2025-24970 - ---- - pom.xml | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/pom.xml b/pom.xml -index 9c201245..4d725e5e 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -559,7 +559,7 @@ - 4.9.0 - 2.2 - 1.5.0 -- 4.1.113.Final -+ 4.1.119.Final - 9.4.57.v20241219 - 2.15.2 - 2.14.6 diff --git a/zookeeper/stackable/patches/3.9.3/patchable.toml b/zookeeper/stackable/patches/3.9.3/patchable.toml deleted file mode 100644 index bcad3c061..000000000 --- a/zookeeper/stackable/patches/3.9.3/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -base = "c26634f34490bb0ea7a09cc51e05ede3b4e320ee" -mirror = "https://github.com/stackabletech/zookeeper.git" From cfce46ae377e8c8566eae90477dcad63d958001c Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 15 Jan 2026 15:31:44 +0100 Subject: [PATCH 10/12] ci(hive): Use Ubicloud runners (#1399) --- .github/workflows/build_hive.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_hive.yaml b/.github/workflows/build_hive.yaml index 4a8cdd9da..5b32d1dc5 100644 --- a/.github/workflows/build_hive.yaml +++ b/.github/workflows/build_hive.yaml @@ -6,7 +6,7 @@ run-name: | on: workflow_dispatch: schedule: - - cron: '0 2 2/2 * *' # https://crontab.guru/#0_2_2/2_*_* + - cron: "0 2 2/2 * *" # https://crontab.guru/#0_2_2/2_*_* push: branches: [main] tags: @@ -36,3 +36,4 @@ jobs: product-name: hive sdp-version: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} registry-namespace: sdp + runners: ubicloud From ec176b161ec6a224cfb4e6c168b73d3394796b27 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 15 Jan 2026 15:32:19 +0100 Subject: [PATCH 11/12] ci(spark): Use Ubicloud runners (#1400) --- .github/workflows/build_spark-k8s.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_spark-k8s.yaml b/.github/workflows/build_spark-k8s.yaml index a549bbc70..5969ddd98 100644 --- a/.github/workflows/build_spark-k8s.yaml +++ b/.github/workflows/build_spark-k8s.yaml @@ -6,7 +6,7 @@ run-name: | on: workflow_dispatch: schedule: - - cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_* + - cron: "0 0 2/2 * *" # https://crontab.guru/#0_0_2/2_*_* push: branches: [main] tags: @@ -36,3 +36,4 @@ jobs: product-name: spark-k8s sdp-version: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} registry-namespace: sdp + runners: ubicloud From b6e741e8a3250f9b67b9b75977369b0c51b8051f Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Sat, 17 Jan 2026 18:37:07 +0100 Subject: [PATCH 12/12] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3073d2f88..887bd38b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project will be documented in this file. - kafka: Add `4.1.1` ([#1395]). - spark: Add `4.1.1` ([#1402]). - spark-connect-client: Add `4.1.1` ([#1402]). +- spark-k8s/hbase-connectors: new image extracted from spark dockerfile ([#1402]). ### Changed