From ca33d8bf2e131801318b0c170b7e312669443026 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Mar 2024 16:49:41 +0530 Subject: [PATCH 01/28] Fix build --- .../org/apache/druid/sql/calcite/CalciteArraysQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 141baa5e5308..2c165ffe3c3b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -1160,7 +1160,7 @@ public void testArrayContainsArrayStringColumns() "SELECT ARRAY_CONTAINS(arrayStringNulls, ARRAY['a', 'b']), ARRAY_CONTAINS(arrayStringNulls, arrayString) FROM druid.arrays LIMIT 5", ImmutableList.of( newScanQueryBuilder() - .dataSource(DATA_SOURCE_ARRAYS) + .dataSource(CalciteTests.ARRAYS_DATASOURCE) .intervals(querySegmentSpec(Filtration.eternity())) .columns("v0", "v1") .virtualColumns( From 61f9b4b435bd7b5e732c4fa0857f34b950b7a002 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 21 Jul 2024 19:44:38 +0530 Subject: [PATCH 02/28] Add IT to test BackwardIncompatibility --- .github/workflows/reusable-revised-its.yml | 36 ++ .github/workflows/revised-its.yml | 42 ++ .../unit-and-integration-tests-unified.yml | 29 +- .../docker-compose.yaml | 107 +++++ .../cases/cluster/Common/dependencies.yaml | 6 +- integration-tests-ex/cases/pom.xml | 9 + .../ITBackwardIncompatibilityIndexerTest.java | 32 ++ .../categories/BackwardIncompatibility.java | 24 ++ .../druid/testsEx/indexer/ITIndexerTest.java | 359 +---------------- .../druid/testsEx/indexer/IndexerTest.java | 380 ++++++++++++++++++ .../BackwardIncompatibility/docker.yaml | 40 ++ integration-tests-ex/image/build-image.sh | 3 + integration-tests-ex/image/docker-build.sh | 18 + integration-tests-ex/image/docker/Dockerfile | 5 +- integration-tests-ex/image/pom.xml | 10 +- 15 files changed, 737 insertions(+), 363 deletions(-) create mode 100644 integration-tests-ex/cases/cluster/BackwardIncompatibility/docker-compose.yaml create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardIncompatibility.java create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/IndexerTest.java create mode 100644 integration-tests-ex/cases/src/test/resources/cluster/BackwardIncompatibility/docker.yaml diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index d9237a52abab..037191dc869f 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -57,6 +57,19 @@ on: AWS_SECRET_ACCESS_KEY: required: false type: string + BACKWARD_INCOMPATIBILITY_IT_ENABLED: + required: false + type: string + default: false + DRUID_PREVIOUS_VERSION: + required: false + type: string + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: + required: false + type: string + DRUID_PREVIOUS_IT_IMAGE_NAME: + required: false + type: string env: MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} # Used by tests to connect to metadata store directly. @@ -106,6 +119,15 @@ jobs: ./druid-container-jdk${{ inputs.build_jdk }}.tar.gz ./integration-tests-ex/image/target/env.sh + - name: Retrieve previous version cached docker image + if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} + id: docker-restore-previous-version + uses: actions/cache/restore@v4 + with: + key: druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }} + path: | + ./druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz + - name: Maven build if: steps.maven-restore.outputs.cache-hit != 'true' || ( steps.docker-restore.outputs.cache-hit != 'true' && steps.targets-restore.outputs.cache-hit != 'true' ) run: | @@ -122,6 +144,15 @@ jobs: echo $DRUID_IT_IMAGE_NAME docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ inputs.build_jdk }}.tar.gz + - name: Save previous version docker image + if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' && (steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true') }} + env: + docker-restore: ${{ toJson(steps.docker-restore.outputs) }} + run: | + docker tag ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }} + echo ${DRUID_PREVIOUS_IT_IMAGE_NAME} + docker save "${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz + - name: Stop and remove docker containers run: | echo "Force stopping all containers and pruning" @@ -133,6 +164,11 @@ jobs: docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz docker images + - name: Load previous version docker image + if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} + run: | + docker load --input druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz + - name: Run IT id: run-it run: ${{ inputs.script }} diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 069562bf7bd3..1cd144a55135 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -18,6 +18,24 @@ name: "Revised ITs workflow" on: workflow_call: + inputs: + BACKWARD_INCOMPATIBILITY_IT_ENABLED: + description: "Flag for backward compatibility IT" + required: false + default: false + type: string + DRUID_PREVIOUS_VERSION: + description: "Previous druid versions to run the test against." + required: false + type: string + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: + description: "URL to download the previous druid version." + required: false + type: string + DRUID_PREVIOUS_IT_IMAGE_NAME: + description: "Druid previous version image name." + required: false + type: string workflow_dispatch: jobs: @@ -62,6 +80,10 @@ jobs: script: ./it.sh github ${{ matrix.it }} it: ${{ matrix.it }} mysql_driver: com.mysql.jdbc.Driver + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} s3-deep-storage-minio: needs: changes @@ -79,3 +101,23 @@ jobs: AWS_REGION: us-east-1 AWS_ACCESS_KEY_ID: admin AWS_SECRET_ACCESS_KEY: miniopassword + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} + + backward-compatibility-it: + needs: changes + uses: ./.github/workflows/reusable-revised-its.yml + if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }} + with: + build_jdk: 8 + runtime_jdk: 8 + use_indexer: MiddleManager + script: ./it.sh github BackwardIncompatibility + it: BackwardIncompatibility + mysql_driver: com.mysql.jdbc.Driver + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 9651a56b8cb7..d9a673c1a056 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -47,6 +47,10 @@ concurrency: env: MYSQL_DRIVER_CLASSNAME: com.mysql.jdbc.Driver # Used to set druid config in docker image for revised ITs SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + BACKWARD_INCOMPATIBILITY_IT_ENABLED: true + DRUID_PREVIOUS_VERSION: 30.0.0 + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz + DRUID_PREVIOUS_IT_IMAGE_NAME: org.apache.druid.integration-tests/test:30.0.0 jobs: build: @@ -111,6 +115,13 @@ jobs: echo $DRUID_IT_IMAGE_NAME docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz + - name: Save previous version docker image + if: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} + run: | + docker tag ${DRUID_PREVIOUS_IT_IMAGE_NAME} ${DRUID_PREVIOUS_IT_IMAGE_NAME}-jdk${{ inputs.build_jdk }}-version${DRUID_PREVIOUS_VERSION} + echo ${DRUID_PREVIOUS_IT_IMAGE_NAME} + docker save "${DRUID_PREVIOUS_IT_IMAGE_NAME}" | gzip > druid-container-jdk${{ inputs.build_jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz + unit-tests-phase2: strategy: fail-fast: false @@ -141,7 +152,23 @@ jobs: if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/standard-its.yml - revised-its: + expose-vars: needs: unit-tests + runs-on: ubuntu-latest + outputs: + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} + steps: + - run: echo "Exposing env vars" + + revised-its: + needs: expose-vars if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml + with: + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ needs.expose-vars.outputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} diff --git a/integration-tests-ex/cases/cluster/BackwardIncompatibility/docker-compose.yaml b/integration-tests-ex/cases/cluster/BackwardIncompatibility/docker-compose.yaml new file mode 100644 index 000000000000..fe71ad2b25b2 --- /dev/null +++ b/integration-tests-ex/cases/cluster/BackwardIncompatibility/docker-compose.yaml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networks: + druid-it-net: + name: druid-it-net + ipam: + config: + - subnet: 172.172.172.0/24 + +services: + zookeeper: + extends: + file: ../Common/dependencies.yaml + service: zookeeper + + metadata: + extends: + file: ../Common/dependencies.yaml + service: metadata + + coordinator: + extends: + file: ../Common/druid.yaml + service: coordinator + image: ${DRUID_PREVIOUS_IT_IMAGE_NAME} + container_name: coordinator + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + # The frequency with which the coordinator polls the database + # for changes. The DB population code has to wait at least this + # long for the coordinator to notice changes. + - druid_manager_segments_pollDuration=PT5S + - druid_coordinator_period=PT10S + depends_on: + - zookeeper + - metadata + + overlord: + extends: + file: ../Common/druid.yaml + service: overlord + image: ${DRUID_PREVIOUS_IT_IMAGE_NAME} + container_name: overlord + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + - metadata + + broker: + extends: + file: ../Common/druid.yaml + service: broker + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + router: + extends: + file: ../Common/druid.yaml + service: router + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + historical: + extends: + file: ../Common/druid.yaml + service: historical + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + middlemanager: + extends: + file: ../Common/druid.yaml + service: middlemanager + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + volumes: + # Test data + - ../../resources:/resources + depends_on: + - zookeeper + + kafka: + extends: + file: ../Common/dependencies.yaml + service: kafka + depends_on: + - zookeeper diff --git a/integration-tests-ex/cases/cluster/Common/dependencies.yaml b/integration-tests-ex/cases/cluster/Common/dependencies.yaml index 0409c30bf538..0adf603ff91e 100644 --- a/integration-tests-ex/cases/cluster/Common/dependencies.yaml +++ b/integration-tests-ex/cases/cluster/Common/dependencies.yaml @@ -26,7 +26,7 @@ services: # See https://hub.docker.com/_/zookeeper zookeeper: # Uncomment the following when running on Apple Silicon processors: - # platform: linux/x86_64 + platform: linux/x86_64 image: zookeeper:${ZK_VERSION} container_name: zookeeper labels: @@ -46,7 +46,7 @@ services: kafka: image: bitnami/kafka:${KAFKA_VERSION} container_name: kafka - # platform: linux/x86_64 + platform: linux/x86_64 labels: druid-int-test: "true" ports: @@ -73,7 +73,7 @@ services: # The image will intialize the user and DB upon first start. metadata: # Uncomment the following when running on Apple Silicon processors: - # platform: linux/x86_64 + platform: linux/x86_64 image: mysql:$MYSQL_IMAGE_VERSION container_name: metadata labels: diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 40461dd8ef17..260a598a4733 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -459,6 +459,15 @@ GcsDeepStorage + + IT-BackwardIncompatibility + + false + + + BackwardIncompatibility + + docker-tests diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java new file mode 100644 index 000000000000..e0578a3a922f --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.BackwardIncompatibility; + +import org.apache.druid.testsEx.categories.BackwardIncompatibility; +import org.apache.druid.testsEx.config.DruidTestRunner; +import org.apache.druid.testsEx.indexer.IndexerTest; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(DruidTestRunner.class) +@Category({BackwardIncompatibility.class}) +public class ITBackwardIncompatibilityIndexerTest extends IndexerTest +{ +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardIncompatibility.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardIncompatibility.java new file mode 100644 index 000000000000..945cc3bedac0 --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardIncompatibility.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.categories; + +public class BackwardIncompatibility +{ +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITIndexerTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITIndexerTest.java index 65b8dc0b1ac0..06a097d608f9 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITIndexerTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/ITIndexerTest.java @@ -19,368 +19,13 @@ package org.apache.druid.testsEx.indexer; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.inject.Inject; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.server.coordinator.CoordinatorDynamicConfig; -import org.apache.druid.testing.clients.CoordinatorResourceTestClient; -import org.apache.druid.testing.utils.ITRetryUtil; import org.apache.druid.testsEx.categories.BatchIndex; import org.apache.druid.testsEx.config.DruidTestRunner; -import org.joda.time.Interval; -import org.junit.Assert; -import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.io.Closeable; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - @RunWith(DruidTestRunner.class) -@Category(BatchIndex.class) -public class ITIndexerTest extends AbstractITBatchIndexTest +@Category({BatchIndex.class}) +public class ITIndexerTest extends IndexerTest { - private static final String INDEX_TASK = "/indexer/wikipedia_index_task.json"; - private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; - private static final String INDEX_DATASOURCE = "wikipedia_index_test"; - - private static final String INDEX_WITH_TIMESTAMP_TASK = "/indexer/wikipedia_with_timestamp_index_task.json"; - // TODO: add queries that validate timestamp is different from the __time column since it is a dimension - // TODO: https://github.com/apache/druid/issues/9565 - private static final String INDEX_WITH_TIMESTAMP_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; - private static final String INDEX_WITH_TIMESTAMP_DATASOURCE = "wikipedia_with_timestamp_index_test"; - - private static final String REINDEX_TASK = "/indexer/wikipedia_reindex_task.json"; - private static final String REINDEX_TASK_WITH_DRUID_INPUT_SOURCE = "/indexer/wikipedia_reindex_druid_input_source_task.json"; - private static final String REINDEX_QUERIES_RESOURCE = "/indexer/wikipedia_reindex_queries.json"; - private static final String REINDEX_DATASOURCE = "wikipedia_reindex_test"; - - private static final String MERGE_INDEX_TASK = "/indexer/wikipedia_merge_index_task.json"; - private static final String MERGE_INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_merge_index_queries.json"; - private static final String MERGE_INDEX_DATASOURCE = "wikipedia_merge_index_test"; - - private static final String MERGE_REINDEX_TASK = "/indexer/wikipedia_merge_reindex_task.json"; - private static final String MERGE_REINDEX_TASK_WITH_DRUID_INPUT_SOURCE = "/indexer/wikipedia_merge_reindex_druid_input_source_task.json"; - private static final String MERGE_REINDEX_QUERIES_RESOURCE = "/indexer/wikipedia_merge_index_queries.json"; - private static final String MERGE_REINDEX_DATASOURCE = "wikipedia_merge_reindex_test"; - - private static final String INDEX_WITH_MERGE_COLUMN_LIMIT_TASK = "/indexer/wikipedia_index_with_merge_column_limit_task.json"; - private static final String INDEX_WITH_MERGE_COLUMN_LIMIT_DATASOURCE = "wikipedia_index_with_merge_column_limit_test"; - - private static final String GET_LOCKED_INTERVALS = "wikipedia_index_get_locked_intervals_test"; - - private static final CoordinatorDynamicConfig DYNAMIC_CONFIG_PAUSED = - CoordinatorDynamicConfig.builder().withPauseCoordination(true).build(); - private static final CoordinatorDynamicConfig DYNAMIC_CONFIG_DEFAULT = - CoordinatorDynamicConfig.builder().build(); - - @Inject - CoordinatorResourceTestClient coordinatorClient; - - @Test - public void testIndexData() throws Exception - { - final String reindexDatasource = REINDEX_DATASOURCE + "-testIndexData"; - final String reindexDatasourceWithDruidInputSource = REINDEX_DATASOURCE + "-testIndexData-druidInputSource"; - try ( - final Closeable ignored1 = unloader(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); - final Closeable ignored2 = unloader(reindexDatasource + config.getExtraDatasourceNameSuffix()); - final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource + config.getExtraDatasourceNameSuffix()) - ) { - - final Function transform = spec -> { - try { - return StringUtils.replace( - spec, - "%%SEGMENT_AVAIL_TIMEOUT_MILLIS%%", - jsonMapper.writeValueAsString("0") - ); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }; - - doIndexTest( - INDEX_DATASOURCE, - INDEX_TASK, - transform, - INDEX_QUERIES_RESOURCE, - false, - true, - true, - new Pair<>(false, false) - ); - doReindexTest( - INDEX_DATASOURCE, - reindexDatasource, - REINDEX_TASK, - REINDEX_QUERIES_RESOURCE, - new Pair<>(false, false) - ); - doReindexTest( - INDEX_DATASOURCE, - reindexDatasourceWithDruidInputSource, - REINDEX_TASK_WITH_DRUID_INPUT_SOURCE, - REINDEX_QUERIES_RESOURCE, - new Pair<>(false, false) - ); - } - } - - @Test - public void testReIndexDataWithTimestamp() throws Exception - { - final String reindexDatasource = REINDEX_DATASOURCE + "-testReIndexDataWithTimestamp"; - final String reindexDatasourceWithDruidInputSource = REINDEX_DATASOURCE + "-testReIndexDataWithTimestamp-druidInputSource"; - try ( - final Closeable ignored1 = unloader(INDEX_WITH_TIMESTAMP_DATASOURCE + config.getExtraDatasourceNameSuffix()); - final Closeable ignored2 = unloader(reindexDatasource + config.getExtraDatasourceNameSuffix()); - final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource + config.getExtraDatasourceNameSuffix()) - ) { - doIndexTest( - INDEX_WITH_TIMESTAMP_DATASOURCE, - INDEX_WITH_TIMESTAMP_TASK, - INDEX_WITH_TIMESTAMP_QUERIES_RESOURCE, - false, - true, - true, - new Pair<>(false, false) - ); - doReindexTest( - INDEX_WITH_TIMESTAMP_DATASOURCE, - reindexDatasource, - REINDEX_TASK, - REINDEX_QUERIES_RESOURCE, - new Pair<>(false, false) - ); - doReindexTest( - INDEX_WITH_TIMESTAMP_DATASOURCE, - reindexDatasourceWithDruidInputSource, - REINDEX_TASK_WITH_DRUID_INPUT_SOURCE, - REINDEX_QUERIES_RESOURCE, - new Pair<>(false, false) - ); - } - } - - @Test - public void testReIndexWithNonExistingDatasource() throws Exception - { - Pair dummyPair = new Pair<>(false, false); - final String fullBaseDatasourceName = "nonExistingDatasource2904"; - final String fullReindexDatasourceName = "newDatasource123"; - - String taskSpec = StringUtils.replace( - getResourceAsString(REINDEX_TASK_WITH_DRUID_INPUT_SOURCE), - "%%DATASOURCE%%", - fullBaseDatasourceName - ); - taskSpec = StringUtils.replace( - taskSpec, - "%%REINDEX_DATASOURCE%%", - fullReindexDatasourceName - ); - - // This method will also verify task is successful after task finish running - // We expect task to be successful even if the datasource to reindex does not exist - submitTaskAndWait( - taskSpec, - fullReindexDatasourceName, - false, - false, - dummyPair - ); - } - - @Test - public void testMERGEIndexData() throws Exception - { - final String reindexDatasource = MERGE_REINDEX_DATASOURCE + "-testMergeIndexData"; - final String reindexDatasourceWithDruidInputSource = MERGE_REINDEX_DATASOURCE + "-testMergeReIndexData-druidInputSource"; - try ( - final Closeable ignored1 = unloader(MERGE_INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); - final Closeable ignored2 = unloader(reindexDatasource + config.getExtraDatasourceNameSuffix()); - final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource + config.getExtraDatasourceNameSuffix()) - ) { - doIndexTest( - MERGE_INDEX_DATASOURCE, - MERGE_INDEX_TASK, - MERGE_INDEX_QUERIES_RESOURCE, - false, - true, - true, - new Pair<>(false, false) - ); - doReindexTest( - MERGE_INDEX_DATASOURCE, - reindexDatasource, - MERGE_REINDEX_TASK, - MERGE_REINDEX_QUERIES_RESOURCE, - new Pair<>(false, false) - ); - doReindexTest( - MERGE_INDEX_DATASOURCE, - reindexDatasourceWithDruidInputSource, - MERGE_REINDEX_TASK_WITH_DRUID_INPUT_SOURCE, - MERGE_INDEX_QUERIES_RESOURCE, - new Pair<>(false, false) - ); - } - } - - /** - * Test that task reports indicate the ingested segments were loaded before the configured timeout expired. - * - * @throws Exception - */ - @Test - public void testIndexDataAwaitSegmentAvailability() throws Exception - { - try ( - final Closeable ignored1 = unloader(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); - ) { - final Function transform = spec -> { - try { - return StringUtils.replace( - spec, - "%%SEGMENT_AVAIL_TIMEOUT_MILLIS%%", - jsonMapper.writeValueAsString("600000") - ); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }; - - doIndexTest( - INDEX_DATASOURCE, - INDEX_TASK, - transform, - INDEX_QUERIES_RESOURCE, - false, - true, - true, - new Pair<>(true, true) - ); - } - } - - /** - * Test that the task still succeeds if the segments do not become available before the configured wait timeout - * expires. - * - * @throws Exception - */ - @Test - public void testIndexDataAwaitSegmentAvailabilityFailsButTaskSucceeds() throws Exception - { - try ( - final Closeable ignored1 = unloader(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); - ) { - coordinatorClient.postDynamicConfig(DYNAMIC_CONFIG_PAUSED); - final Function transform = spec -> { - try { - return StringUtils.replace( - spec, - "%%SEGMENT_AVAIL_TIMEOUT_MILLIS%%", - jsonMapper.writeValueAsString("1") - ); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }; - - doIndexTest( - INDEX_DATASOURCE, - INDEX_TASK, - transform, - INDEX_QUERIES_RESOURCE, - false, - false, - false, - new Pair<>(true, false) - ); - coordinatorClient.postDynamicConfig(DYNAMIC_CONFIG_DEFAULT); - ITRetryUtil.retryUntilTrue( - () -> coordinator.areSegmentsLoaded(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()), "Segment Load" - ); - } - } - - - @Test - public void testIndexWithMergeColumnLimitData() throws Exception - { - try ( - final Closeable ignored1 = unloader(INDEX_WITH_MERGE_COLUMN_LIMIT_DATASOURCE + config.getExtraDatasourceNameSuffix()); - ) { - doIndexTest( - INDEX_WITH_MERGE_COLUMN_LIMIT_DATASOURCE, - INDEX_WITH_MERGE_COLUMN_LIMIT_TASK, - INDEX_QUERIES_RESOURCE, - false, - true, - true, - new Pair<>(false, false) - ); - } - } - - @Test - public void testGetLockedIntervals() throws Exception - { - final String datasourceName = GET_LOCKED_INTERVALS + config.getExtraDatasourceNameSuffix(); - try (final Closeable ignored = unloader(datasourceName)) { - // Submit an Indexing Task - submitIndexTask(INDEX_TASK, datasourceName); - - // Wait until it acquires a lock - final Map minTaskPriority = Collections.singletonMap(datasourceName, 0); - final Map> lockedIntervals = new HashMap<>(); - ITRetryUtil.retryUntilFalse( - () -> { - lockedIntervals.clear(); - lockedIntervals.putAll(indexer.getLockedIntervals(minTaskPriority)); - return lockedIntervals.isEmpty(); - }, - "Verify Intervals are Locked" - ); - - // Verify the locked intervals for this datasource - Assert.assertEquals(lockedIntervals.size(), 1); - Assert.assertEquals( - lockedIntervals.get(datasourceName), - Collections.singletonList(Intervals.of("2013-08-31/2013-09-02")) - ); - - ITRetryUtil.retryUntilTrue( - () -> coordinator.areSegmentsLoaded(datasourceName), - "Segment Load" - ); - } - } - - @Test - public void testJsonFunctions() throws Exception - { - final String taskSpec = getResourceAsString("/indexer/json_path_index_task.json"); - - submitTaskAndWait( - taskSpec, - "json_path_index_test", - false, - true, - new Pair<>(false, false) - ); - - doTestQuery("json_path_index_test", "/indexer/json_path_index_queries.json"); - } } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/IndexerTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/IndexerTest.java new file mode 100644 index 000000000000..ff30110f8ae1 --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/IndexerTest.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.indexer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.inject.Inject; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.server.coordinator.CoordinatorDynamicConfig; +import org.apache.druid.testing.clients.CoordinatorResourceTestClient; +import org.apache.druid.testing.utils.ITRetryUtil; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Test; + +import java.io.Closeable; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class IndexerTest extends AbstractITBatchIndexTest +{ + private static final String INDEX_TASK = "/indexer/wikipedia_index_task.json"; + private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; + private static final String INDEX_DATASOURCE = "wikipedia_index_test"; + + private static final String INDEX_WITH_TIMESTAMP_TASK = "/indexer/wikipedia_with_timestamp_index_task.json"; + // TODO: add queries that validate timestamp is different from the __time column since it is a dimension + // TODO: https://github.com/apache/druid/issues/9565 + private static final String INDEX_WITH_TIMESTAMP_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; + private static final String INDEX_WITH_TIMESTAMP_DATASOURCE = "wikipedia_with_timestamp_index_test"; + + private static final String REINDEX_TASK = "/indexer/wikipedia_reindex_task.json"; + private static final String REINDEX_TASK_WITH_DRUID_INPUT_SOURCE = "/indexer/wikipedia_reindex_druid_input_source_task.json"; + private static final String REINDEX_QUERIES_RESOURCE = "/indexer/wikipedia_reindex_queries.json"; + private static final String REINDEX_DATASOURCE = "wikipedia_reindex_test"; + + private static final String MERGE_INDEX_TASK = "/indexer/wikipedia_merge_index_task.json"; + private static final String MERGE_INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_merge_index_queries.json"; + private static final String MERGE_INDEX_DATASOURCE = "wikipedia_merge_index_test"; + + private static final String MERGE_REINDEX_TASK = "/indexer/wikipedia_merge_reindex_task.json"; + private static final String MERGE_REINDEX_TASK_WITH_DRUID_INPUT_SOURCE = "/indexer/wikipedia_merge_reindex_druid_input_source_task.json"; + private static final String MERGE_REINDEX_QUERIES_RESOURCE = "/indexer/wikipedia_merge_index_queries.json"; + private static final String MERGE_REINDEX_DATASOURCE = "wikipedia_merge_reindex_test"; + + private static final String INDEX_WITH_MERGE_COLUMN_LIMIT_TASK = "/indexer/wikipedia_index_with_merge_column_limit_task.json"; + private static final String INDEX_WITH_MERGE_COLUMN_LIMIT_DATASOURCE = "wikipedia_index_with_merge_column_limit_test"; + + private static final String GET_LOCKED_INTERVALS = "wikipedia_index_get_locked_intervals_test"; + + private static final CoordinatorDynamicConfig DYNAMIC_CONFIG_PAUSED = + CoordinatorDynamicConfig.builder().withPauseCoordination(true).build(); + private static final CoordinatorDynamicConfig DYNAMIC_CONFIG_DEFAULT = + CoordinatorDynamicConfig.builder().build(); + + @Inject + CoordinatorResourceTestClient coordinatorClient; + + @Test + public void testIndexData() throws Exception + { + final String reindexDatasource = REINDEX_DATASOURCE + "-testIndexData"; + final String reindexDatasourceWithDruidInputSource = REINDEX_DATASOURCE + "-testIndexData-druidInputSource"; + try ( + final Closeable ignored1 = unloader(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); + final Closeable ignored2 = unloader(reindexDatasource + config.getExtraDatasourceNameSuffix()); + final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource + config.getExtraDatasourceNameSuffix()) + ) { + + final Function transform = spec -> { + try { + return StringUtils.replace( + spec, + "%%SEGMENT_AVAIL_TIMEOUT_MILLIS%%", + jsonMapper.writeValueAsString("0") + ); + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }; + + doIndexTest( + INDEX_DATASOURCE, + INDEX_TASK, + transform, + INDEX_QUERIES_RESOURCE, + false, + true, + true, + new Pair<>(false, false) + ); + doReindexTest( + INDEX_DATASOURCE, + reindexDatasource, + REINDEX_TASK, + REINDEX_QUERIES_RESOURCE, + new Pair<>(false, false) + ); + doReindexTest( + INDEX_DATASOURCE, + reindexDatasourceWithDruidInputSource, + REINDEX_TASK_WITH_DRUID_INPUT_SOURCE, + REINDEX_QUERIES_RESOURCE, + new Pair<>(false, false) + ); + } + } + + @Test + public void testReIndexDataWithTimestamp() throws Exception + { + final String reindexDatasource = REINDEX_DATASOURCE + "-testReIndexDataWithTimestamp"; + final String reindexDatasourceWithDruidInputSource = REINDEX_DATASOURCE + "-testReIndexDataWithTimestamp-druidInputSource"; + try ( + final Closeable ignored1 = unloader(INDEX_WITH_TIMESTAMP_DATASOURCE + config.getExtraDatasourceNameSuffix()); + final Closeable ignored2 = unloader(reindexDatasource + config.getExtraDatasourceNameSuffix()); + final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource + config.getExtraDatasourceNameSuffix()) + ) { + doIndexTest( + INDEX_WITH_TIMESTAMP_DATASOURCE, + INDEX_WITH_TIMESTAMP_TASK, + INDEX_WITH_TIMESTAMP_QUERIES_RESOURCE, + false, + true, + true, + new Pair<>(false, false) + ); + doReindexTest( + INDEX_WITH_TIMESTAMP_DATASOURCE, + reindexDatasource, + REINDEX_TASK, + REINDEX_QUERIES_RESOURCE, + new Pair<>(false, false) + ); + doReindexTest( + INDEX_WITH_TIMESTAMP_DATASOURCE, + reindexDatasourceWithDruidInputSource, + REINDEX_TASK_WITH_DRUID_INPUT_SOURCE, + REINDEX_QUERIES_RESOURCE, + new Pair<>(false, false) + ); + } + } + + @Test + public void testReIndexWithNonExistingDatasource() throws Exception + { + Pair dummyPair = new Pair<>(false, false); + final String fullBaseDatasourceName = "nonExistingDatasource2904"; + final String fullReindexDatasourceName = "newDatasource123"; + + String taskSpec = StringUtils.replace( + getResourceAsString(REINDEX_TASK_WITH_DRUID_INPUT_SOURCE), + "%%DATASOURCE%%", + fullBaseDatasourceName + ); + taskSpec = StringUtils.replace( + taskSpec, + "%%REINDEX_DATASOURCE%%", + fullReindexDatasourceName + ); + + // This method will also verify task is successful after task finish running + // We expect task to be successful even if the datasource to reindex does not exist + submitTaskAndWait( + taskSpec, + fullReindexDatasourceName, + false, + false, + dummyPair + ); + } + + @Test + public void testMERGEIndexData() throws Exception + { + final String reindexDatasource = MERGE_REINDEX_DATASOURCE + "-testMergeIndexData"; + final String reindexDatasourceWithDruidInputSource = MERGE_REINDEX_DATASOURCE + "-testMergeReIndexData-druidInputSource"; + try ( + final Closeable ignored1 = unloader(MERGE_INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); + final Closeable ignored2 = unloader(reindexDatasource + config.getExtraDatasourceNameSuffix()); + final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource + config.getExtraDatasourceNameSuffix()) + ) { + doIndexTest( + MERGE_INDEX_DATASOURCE, + MERGE_INDEX_TASK, + MERGE_INDEX_QUERIES_RESOURCE, + false, + true, + true, + new Pair<>(false, false) + ); + doReindexTest( + MERGE_INDEX_DATASOURCE, + reindexDatasource, + MERGE_REINDEX_TASK, + MERGE_REINDEX_QUERIES_RESOURCE, + new Pair<>(false, false) + ); + doReindexTest( + MERGE_INDEX_DATASOURCE, + reindexDatasourceWithDruidInputSource, + MERGE_REINDEX_TASK_WITH_DRUID_INPUT_SOURCE, + MERGE_INDEX_QUERIES_RESOURCE, + new Pair<>(false, false) + ); + } + } + + /** + * Test that task reports indicate the ingested segments were loaded before the configured timeout expired. + * + * @throws Exception + */ + @Test + public void testIndexDataAwaitSegmentAvailability() throws Exception + { + try ( + final Closeable ignored1 = unloader(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); + ) { + final Function transform = spec -> { + try { + return StringUtils.replace( + spec, + "%%SEGMENT_AVAIL_TIMEOUT_MILLIS%%", + jsonMapper.writeValueAsString("600000") + ); + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }; + + doIndexTest( + INDEX_DATASOURCE, + INDEX_TASK, + transform, + INDEX_QUERIES_RESOURCE, + false, + true, + true, + new Pair<>(true, true) + ); + } + } + + /** + * Test that the task still succeeds if the segments do not become available before the configured wait timeout + * expires. + * + * @throws Exception + */ + @Test + public void testIndexDataAwaitSegmentAvailabilityFailsButTaskSucceeds() throws Exception + { + try ( + final Closeable ignored1 = unloader(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()); + ) { + coordinatorClient.postDynamicConfig(DYNAMIC_CONFIG_PAUSED); + final Function transform = spec -> { + try { + return StringUtils.replace( + spec, + "%%SEGMENT_AVAIL_TIMEOUT_MILLIS%%", + jsonMapper.writeValueAsString("1") + ); + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }; + + doIndexTest( + INDEX_DATASOURCE, + INDEX_TASK, + transform, + INDEX_QUERIES_RESOURCE, + false, + false, + false, + new Pair<>(true, false) + ); + coordinatorClient.postDynamicConfig(DYNAMIC_CONFIG_DEFAULT); + ITRetryUtil.retryUntilTrue( + () -> coordinator.areSegmentsLoaded(INDEX_DATASOURCE + config.getExtraDatasourceNameSuffix()), "Segment Load" + ); + } + } + + + @Test + public void testIndexWithMergeColumnLimitData() throws Exception + { + try ( + final Closeable ignored1 = unloader(INDEX_WITH_MERGE_COLUMN_LIMIT_DATASOURCE + config.getExtraDatasourceNameSuffix()); + ) { + doIndexTest( + INDEX_WITH_MERGE_COLUMN_LIMIT_DATASOURCE, + INDEX_WITH_MERGE_COLUMN_LIMIT_TASK, + INDEX_QUERIES_RESOURCE, + false, + true, + true, + new Pair<>(false, false) + ); + } + } + + @Test + public void testGetLockedIntervals() throws Exception + { + final String datasourceName = GET_LOCKED_INTERVALS + config.getExtraDatasourceNameSuffix(); + try (final Closeable ignored = unloader(datasourceName)) { + // Submit an Indexing Task + submitIndexTask(INDEX_TASK, datasourceName); + + // Wait until it acquires a lock + final Map minTaskPriority = Collections.singletonMap(datasourceName, 0); + final Map> lockedIntervals = new HashMap<>(); + ITRetryUtil.retryUntilFalse( + () -> { + lockedIntervals.clear(); + lockedIntervals.putAll(indexer.getLockedIntervals(minTaskPriority)); + return lockedIntervals.isEmpty(); + }, + "Verify Intervals are Locked" + ); + + // Verify the locked intervals for this datasource + Assert.assertEquals(lockedIntervals.size(), 1); + Assert.assertEquals( + lockedIntervals.get(datasourceName), + Collections.singletonList(Intervals.of("2013-08-31/2013-09-02")) + ); + + ITRetryUtil.retryUntilTrue( + () -> coordinator.areSegmentsLoaded(datasourceName), + "Segment Load" + ); + } + } + + @Test + public void testJsonFunctions() throws Exception + { + final String taskSpec = getResourceAsString("/indexer/json_path_index_task.json"); + + submitTaskAndWait( + taskSpec, + "json_path_index_test", + false, + true, + new Pair<>(false, false) + ); + + doTestQuery("json_path_index_test", "/indexer/json_path_index_queries.json"); + } +} diff --git a/integration-tests-ex/cases/src/test/resources/cluster/BackwardIncompatibility/docker.yaml b/integration-tests-ex/cases/src/test/resources/cluster/BackwardIncompatibility/docker.yaml new file mode 100644 index 000000000000..d676f530e908 --- /dev/null +++ b/integration-tests-ex/cases/src/test/resources/cluster/BackwardIncompatibility/docker.yaml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#------------------------------------------------------------------------- + +# Definition of the batch index test cluster. +# See https://yaml.org/spec/1.2.2 for more about YAML +include: + - /cluster/Common/zk-metastore.yaml + +druid: + coordinator: + instances: + - port: 8081 + overlord: + instances: + - port: 8090 + broker: + instances: + - port: 8082 + router: + instances: + - port: 8888 + historical: + instances: + - port: 8083 + indexer: + instances: + - port: 8091 diff --git a/integration-tests-ex/image/build-image.sh b/integration-tests-ex/image/build-image.sh index 4a31a7841944..1b35dc14e769 100755 --- a/integration-tests-ex/image/build-image.sh +++ b/integration-tests-ex/image/build-image.sh @@ -43,6 +43,9 @@ export CONFLUENT_VERSION=$CONFLUENT_VERSION export MARIADB_VERSION=$MARIADB_VERSION export HADOOP_VERSION=$HADOOP_VERSION export DRUID_IT_IMAGE_NAME=$DRUID_IT_IMAGE_NAME +export DRUID_PREVIOUS_VERSION=$DRUID_PREVIOUS_VERSION +export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=$DRUID_PREVIOUS_VERSION_DOWNLOAD_URL +export DRUID_PREVIOUS_IT_IMAGE_NAME=$DRUID_PREVIOUS_IT_IMAGE_NAME EOF exec bash $SCRIPT_DIR/docker-build.sh diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index 6a945aa6129a..fcd4d1c80f34 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -53,4 +53,22 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg CONFLUENT_VERSION=$CONFLUENT_VERSION \ --build-arg HADOOP_VERSION=$HADOOP_VERSION \ --build-arg MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME \ + --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ + . + +if [ $BACKWARD_COMPATIBILITY_IT_ENABLED != "true" ]; then + exit 1 +fi + +# Download the previous druid tar +curl -L $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL --output apache-druid-$DRUID_PREVIOUS_VERSION-bin.tar.gz + +docker build -t $DRUID_PREVIOUS_IT_IMAGE_NAME \ + --build-arg DRUID_VERSION=$DRUID_PREVIOUS_VERSION \ + --build-arg MYSQL_VERSION=$MYSQL_VERSION \ + --build-arg MARIADB_VERSION=$MARIADB_VERSION \ + --build-arg CONFLUENT_VERSION=$CONFLUENT_VERSION \ + --build-arg HADOOP_VERSION=$HADOOP_VERSION \ + --build-arg MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME \ + --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index a77a5c2d023e..99b2da9476b2 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -46,13 +46,15 @@ ARG MARIADB_VERSION ENV MARIADB_VERSION=$MARIADB_VERSION ARG MYSQL_DRIVER_CLASSNAME=com.mysql.jdbc.Driver ENV MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME +ARG DRUID_TESTING_TOOLS_VERSION +ENV DRUID_TESTING_TOOLS_VERSION=$DRUID_TESTING_TOOLS_VERSION ENV DRUID_HOME=/usr/local/druid # Populate build artifacts COPY apache-druid-${DRUID_VERSION}-bin.tar.gz /usr/local/ -COPY druid-it-tools-${DRUID_VERSION}.jar /tmp/druid/extensions/druid-it-tools/ +COPY druid-it-tools-${DRUID_TESTING_TOOLS_VERSION}.jar /tmp/druid/extensions/druid-it-tools/ COPY kafka-protobuf-provider-${CONFLUENT_VERSION}.jar /tmp/druid/lib/ COPY mysql-connector-j-${MYSQL_VERSION}.jar /tmp/druid/lib/ COPY mariadb-java-client-${MARIADB_VERSION}.jar /tmp/druid/lib/ @@ -71,3 +73,4 @@ RUN bash /test-setup.sh USER druid:druid WORKDIR / ENTRYPOINT [ "bash", "/launch.sh" ] + diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index 600501edb92b..75eb33beed35 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -54,7 +54,11 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven - ${project.groupId}/test:${project.version} + 5.5.1 2.7.3 5.7-debian @@ -209,6 +213,10 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven ${hadoop.compile.version} ${project.version} ${druid.it.image-name} + ${druid.it.prev.version} + ${druid.it.prev.version.download.url} + ${druid.it.prev.image-name} + ${druid.it.backward-compatibility.enabled} ${project.build.directory} From a58e7edacb4bf9d8f65ec634a400e7d273cf16b3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 21 Jul 2024 19:47:57 +0530 Subject: [PATCH 03/28] minor change --- integration-tests-ex/image/docker-build.sh | 2 +- integration-tests-ex/image/pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index fcd4d1c80f34..eb7037981853 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -56,7 +56,7 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . -if [ $BACKWARD_COMPATIBILITY_IT_ENABLED != "true" ]; then +if [ $BACKWARD_INCOMPATIBILITY_IT_ENABLED != "true" ]; then exit 1 fi diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index 75eb33beed35..a9815dad6ca8 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -58,7 +58,7 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven 30.0.0 https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz ${project.groupId}/test:${druid.it.prev.version} - true --> + true --> 5.5.1 2.7.3 5.7-debian @@ -216,7 +216,7 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven ${druid.it.prev.version} ${druid.it.prev.version.download.url} ${druid.it.prev.image-name} - ${druid.it.backward-compatibility.enabled} + ${druid.it.backward-incompatibility.enabled} ${project.build.directory} From c41cdebbb4abd3bc6cc55442006ad584a015a652 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 21 Jul 2024 22:01:04 +0530 Subject: [PATCH 04/28] test --- integration-tests-ex/image/docker-build.sh | 7 +++++++ integration-tests-ex/image/pom.xml | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index eb7037981853..921fec3bd499 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -56,6 +56,13 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . +echo "printing env vars" + +echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED +echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL +echo $DRUID_PREVIOUS_IT_IMAGE_NAME +echo $DRUID_PREVIOUS_VERSION + if [ $BACKWARD_INCOMPATIBILITY_IT_ENABLED != "true" ]; then exit 1 fi diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index a9815dad6ca8..a32538314446 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -54,11 +54,11 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven - + false 5.5.1 2.7.3 5.7-debian From b08cf00d85d6fa0a00532345b3f318376e744537 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 21 Jul 2024 22:04:19 +0530 Subject: [PATCH 05/28] test --- integration-tests-ex/image/build-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests-ex/image/build-image.sh b/integration-tests-ex/image/build-image.sh index 1b35dc14e769..954061cb669e 100755 --- a/integration-tests-ex/image/build-image.sh +++ b/integration-tests-ex/image/build-image.sh @@ -43,6 +43,7 @@ export CONFLUENT_VERSION=$CONFLUENT_VERSION export MARIADB_VERSION=$MARIADB_VERSION export HADOOP_VERSION=$HADOOP_VERSION export DRUID_IT_IMAGE_NAME=$DRUID_IT_IMAGE_NAME +export BACKWARD_INCOMPATIBILITY_IT_ENABLED=$BACKWARD_INCOMPATIBILITY_IT_ENABLED export DRUID_PREVIOUS_VERSION=$DRUID_PREVIOUS_VERSION export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=$DRUID_PREVIOUS_VERSION_DOWNLOAD_URL export DRUID_PREVIOUS_IT_IMAGE_NAME=$DRUID_PREVIOUS_IT_IMAGE_NAME From 00b8f29e837ad372176df366ed40242e68265c18 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 21 Jul 2024 22:18:03 +0530 Subject: [PATCH 06/28] disable some gha checks temporarily --- .github/workflows/codeql.yml | 33 ++++++++++-------- .github/workflows/distribution-checks.yml | 34 ++++++++++--------- .github/workflows/static-checks.yml | 22 ++++++------ .../unit-and-integration-tests-unified.yml | 11 +++--- 4 files changed, 55 insertions(+), 45 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7cfeb63cbff0..e4cd90de85e5 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,21 +1,24 @@ name: "CodeQL" on: - push: - paths-ignore: - - '**/*.md' - - 'dev/**' - - 'docs/**' - branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ] - pull_request: - paths-ignore: - - '**/*.md' - - 'dev/**' - - 'docs/**' - # The branches below must be a subset of the branches above - branches: [ 'master' ] - schedule: - - cron: '18 15 * * 4' + workflow_call: + +#on: +# push: +# paths-ignore: +# - '**/*.md' +# - 'dev/**' +# - 'docs/**' +# branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ] +# pull_request: +# paths-ignore: +# - '**/*.md' +# - 'dev/**' +# - 'docs/**' +# # The branches below must be a subset of the branches above +# branches: [ 'master' ] +# schedule: +# - cron: '18 15 * * 4' jobs: analyze: diff --git a/.github/workflows/distribution-checks.yml b/.github/workflows/distribution-checks.yml index caadb16b608e..feb188eb3d04 100644 --- a/.github/workflows/distribution-checks.yml +++ b/.github/workflows/distribution-checks.yml @@ -15,22 +15,24 @@ name: "Distribution Checks" on: - push: - branches: - - master - - '[0-9]+.[0-9]+.[0-9]+' # release branches - - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches - paths: - - 'distribution/**' - - '**/pom.xml' - pull_request: - branches: - - master - - '[0-9]+.[0-9]+.[0-9]+' # release branches - - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches - paths: - - 'distribution/**' - - '**/pom.xml' + workflow_call: + +# push: +# branches: +# - master +# - '[0-9]+.[0-9]+.[0-9]+' # release branches +# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches +# paths: +# - 'distribution/**' +# - '**/pom.xml' +# pull_request: +# branches: +# - master +# - '[0-9]+.[0-9]+.[0-9]+' # release branches +# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches +# paths: +# - 'distribution/**' +# - '**/pom.xml' jobs: docker-build: diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index a374cf72ccfe..82bef910cde8 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -15,16 +15,18 @@ name: "Static Checks CI" on: - push: - branches: - - master - - '[0-9]+.[0-9]+.[0-9]+' # release branches - - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches - pull_request: - branches: - - master - - '[0-9]+.[0-9]+.[0-9]+' # release branches - - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + workflow_call: + +# push: +# branches: +# - master +# - '[0-9]+.[0-9]+.[0-9]+' # release branches +# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches +# pull_request: +# branches: +# - master +# - '[0-9]+.[0-9]+.[0-9]+' # release branches +# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.run_id }}' # group workflows only on pull_requests and not on branch commits diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index d9a673c1a056..856329646973 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -58,7 +58,8 @@ jobs: strategy: fail-fast: false matrix: - jdk: [ '8', '11', '17', '21' ] + #jdk: [ '8', '11', '17', '21' ] + jdK: [ '8' ] runs-on: ubuntu-latest steps: - name: Checkout branch @@ -130,12 +131,13 @@ jobs: name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests - if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} with: jdk: ${{ matrix.jdk }} sql_compatibility: true unit-tests: + if: false strategy: fail-fast: false matrix: @@ -149,11 +151,12 @@ jobs: standard-its: needs: unit-tests - if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/standard-its.yml expose-vars: - needs: unit-tests + #needs: unit-tests + needs: build runs-on: ubuntu-latest outputs: BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} From a351a4880439239f1e660c5aeef16b8f647c30b4 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 21 Jul 2024 22:53:37 +0530 Subject: [PATCH 07/28] Set env while running revised it --- .github/workflows/reusable-revised-its.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index 037191dc869f..1ef054e917a2 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -171,6 +171,12 @@ jobs: - name: Run IT id: run-it + env: + MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} run: ${{ inputs.script }} - name: Collect docker logs on failure From ccad637ecd964dba5355eb6806f12d8f7d79983f Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 21 Jul 2024 23:58:13 +0530 Subject: [PATCH 08/28] another way to set env --- .github/workflows/reusable-revised-its.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index 1ef054e917a2..02ab3eef6387 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -171,13 +171,12 @@ jobs: - name: Run IT id: run-it - env: - MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} - run: ${{ inputs.script }} + run: | + echo "BACKWARD_INCOMPATIBILITY_IT_ENABLED=${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }}" >> $GITHUB_ENV + echo "DRUID_PREVIOUS_VERSION=${{ inputs.DRUID_PREVIOUS_VERSION }}" >> $GITHUB_ENV + echo "DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}" >> $GITHUB_ENV + echo "DRUID_PREVIOUS_IT_IMAGE_NAME=${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" >> $GITHUB_ENV + ${{ inputs.script }} - name: Collect docker logs on failure if: ${{ failure() && steps.run-it.conclusion == 'failure' }} From bfad1e4ae2e75d13cf852ea7f842c56390306f9e Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 00:52:24 +0530 Subject: [PATCH 09/28] log envs in it.sh --- it.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/it.sh b/it.sh index 12519c24b54d..1b4a6e6cb953 100755 --- a/it.sh +++ b/it.sh @@ -286,10 +286,17 @@ case $CMD in tail_logs ;; "github" ) + echo "it.sh printing env vars" + echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED + echo $DRUID_PREVIOUS_VERSION + echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL + echo $DRUID_PREVIOUS_IT_IMAGE_NAME + set +e $0 test $CATEGORY RESULT=$? + # Include logs, but only for failures. if [ $RESULT -ne 0 ]; then $0 tail $CATEGORY From b290f87bb77eb184326dd1a13bdbeb77e1933587 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 01:06:48 +0530 Subject: [PATCH 10/28] test --- .github/workflows/unit-and-integration-tests-unified.yml | 7 +++++++ it.sh | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 856329646973..c09536a8a5ec 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -101,10 +101,17 @@ jobs: - name: Maven build id: maven_build + if: false run: | ./it.sh ci - name: Container build + env: + MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} run: | ./it.sh image source ./integration-tests-ex/image/target/env.sh diff --git a/it.sh b/it.sh index 1b4a6e6cb953..8c36c1dad7b8 100755 --- a/it.sh +++ b/it.sh @@ -241,6 +241,11 @@ case $CMD in mvn -B install -pl :druid-it-tools ;; "image" ) + echo "it.sh printing env vars" + echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED + echo $DRUID_PREVIOUS_VERSION + echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL + echo $DRUID_PREVIOUS_IT_IMAGE_NAME cd $DRUID_DEV/integration-tests-ex/image mvn -B install -P test-image $MAVEN_IGNORE ;; From 666609e5b3b8924342bc943851e000e1546eb734 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 01:15:17 +0530 Subject: [PATCH 11/28] fix minor bug --- .../workflows/unit-and-integration-tests-unified.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index c09536a8a5ec..bdb98be9acd9 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -107,11 +107,11 @@ jobs: - name: Container build env: - MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} + MYSQL_DRIVER_CLASSNAME: ${{ env.mysql_driver }} + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} run: | ./it.sh image source ./integration-tests-ex/image/target/env.sh From a5e4863b6df30c9e1aa4127a2abd9c9f0d1ae79c Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 01:19:42 +0530 Subject: [PATCH 12/28] pass env vars in resusable-revised-its --- .github/workflows/reusable-revised-its.yml | 13 +++++++++---- .../unit-and-integration-tests-unified.yml | 1 - 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index 02ab3eef6387..ee948cd9f3c5 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -137,6 +137,10 @@ jobs: if: steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true' env: docker-restore: ${{ toJson(steps.docker-restore.outputs) }} + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} run: | ./it.sh image source ./integration-tests-ex/image/target/env.sh @@ -171,11 +175,12 @@ jobs: - name: Run IT id: run-it + env: + BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} run: | - echo "BACKWARD_INCOMPATIBILITY_IT_ENABLED=${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }}" >> $GITHUB_ENV - echo "DRUID_PREVIOUS_VERSION=${{ inputs.DRUID_PREVIOUS_VERSION }}" >> $GITHUB_ENV - echo "DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}" >> $GITHUB_ENV - echo "DRUID_PREVIOUS_IT_IMAGE_NAME=${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" >> $GITHUB_ENV ${{ inputs.script }} - name: Collect docker logs on failure diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index bdb98be9acd9..51392e563cba 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -107,7 +107,6 @@ jobs: - name: Container build env: - MYSQL_DRIVER_CLASSNAME: ${{ env.mysql_driver }} BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} From 0616c79ceb82687b9f5549d47c2d6d03e6ca958d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 01:36:33 +0530 Subject: [PATCH 13/28] enable maven build again --- .github/workflows/unit-and-integration-tests-unified.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 51392e563cba..28b1b1a4473d 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -101,7 +101,6 @@ jobs: - name: Maven build id: maven_build - if: false run: | ./it.sh ci From 97486b5828216293fecd9dbb236dc94551493b2b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 11:02:05 +0530 Subject: [PATCH 14/28] remove env override from pom --- integration-tests-ex/image/build-image.sh | 5 ----- integration-tests-ex/image/docker-build.sh | 13 ++++--------- integration-tests-ex/image/pom.xml | 8 ++++---- it.sh | 4 ++++ 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/integration-tests-ex/image/build-image.sh b/integration-tests-ex/image/build-image.sh index 954061cb669e..65bd816b1248 100755 --- a/integration-tests-ex/image/build-image.sh +++ b/integration-tests-ex/image/build-image.sh @@ -32,7 +32,6 @@ SCRIPT_DIR=$(cd $(dirname $0) && pwd) # Copy environment variables to a file. Used for manual rebuilds # and by scripts that start the test cluster. - cat > $TARGET_DIR/env.sh << EOF export ZK_VERSION=$ZK_VERSION export KAFKA_VERSION=$KAFKA_VERSION @@ -43,10 +42,6 @@ export CONFLUENT_VERSION=$CONFLUENT_VERSION export MARIADB_VERSION=$MARIADB_VERSION export HADOOP_VERSION=$HADOOP_VERSION export DRUID_IT_IMAGE_NAME=$DRUID_IT_IMAGE_NAME -export BACKWARD_INCOMPATIBILITY_IT_ENABLED=$BACKWARD_INCOMPATIBILITY_IT_ENABLED -export DRUID_PREVIOUS_VERSION=$DRUID_PREVIOUS_VERSION -export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=$DRUID_PREVIOUS_VERSION_DOWNLOAD_URL -export DRUID_PREVIOUS_IT_IMAGE_NAME=$DRUID_PREVIOUS_IT_IMAGE_NAME EOF exec bash $SCRIPT_DIR/docker-build.sh diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index 921fec3bd499..d5351a3aa2e0 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -56,15 +56,10 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . -echo "printing env vars" - -echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED -echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL -echo $DRUID_PREVIOUS_IT_IMAGE_NAME -echo $DRUID_PREVIOUS_VERSION - -if [ $BACKWARD_INCOMPATIBILITY_IT_ENABLED != "true" ]; then - exit 1 +echo "val is $BACKWARD_INCOMPATIBILITY_IT_ENABLED" +if [[ -z "${BACKWARD_INCOMPATIBILITY_IT_ENABLED:-""}" || $BACKWARD_INCOMPATIBILITY_IT_ENABLED != "true" ]]; then + echo "Not building previous version image." + exit 0 fi # Download the previous druid tar diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index a32538314446..6467152819c4 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -55,10 +55,10 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven image name will typically be provided by the build enfironment, and will override this default name. --> ${project.groupId}/test:${project.version} - 27.0.0 + 5.5.1 2.7.3 5.7-debian @@ -213,10 +213,10 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven ${hadoop.compile.version} ${project.version} ${druid.it.image-name} - ${druid.it.prev.version} + ${project.build.directory} diff --git a/it.sh b/it.sh index 8c36c1dad7b8..c8d3a74f8357 100755 --- a/it.sh +++ b/it.sh @@ -242,6 +242,10 @@ case $CMD in ;; "image" ) echo "it.sh printing env vars" + export DRUID_PREVIOUS_VERSION=30.0.0 + export BACKWARD_INCOMPATIBILITY_IT_ENABLED=true + export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz + export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:30.0.0 echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED echo $DRUID_PREVIOUS_VERSION echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL From 4df32961a486cc9eb18e28355e8a2263020a3fad Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 11:39:15 +0530 Subject: [PATCH 15/28] temporarily remove conditional --- .github/workflows/unit-and-integration-tests-unified.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 28b1b1a4473d..7a8251e0afd8 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -173,7 +173,7 @@ jobs: revised-its: needs: expose-vars - if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: true #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml with: BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ needs.expose-vars.outputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} From 4d97b550b002cf20e3d3dac72cfa0e33f98d9be2 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 14:09:31 +0530 Subject: [PATCH 16/28] use correct jdk version in docker tag step --- .github/workflows/reusable-revised-its.yml | 13 +++++++------ .../unit-and-integration-tests-unified.yml | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index ee948cd9f3c5..254536e66944 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -120,13 +120,13 @@ jobs: ./integration-tests-ex/image/target/env.sh - name: Retrieve previous version cached docker image - if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} id: docker-restore-previous-version + if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} uses: actions/cache/restore@v4 with: - key: druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }} + key: druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }} path: | - ./druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz + ./druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz - name: Maven build if: steps.maven-restore.outputs.cache-hit != 'true' || ( steps.docker-restore.outputs.cache-hit != 'true' && steps.targets-restore.outputs.cache-hit != 'true' ) @@ -153,9 +153,9 @@ jobs: env: docker-restore: ${{ toJson(steps.docker-restore.outputs) }} run: | - docker tag ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }} + docker tag ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }} echo ${DRUID_PREVIOUS_IT_IMAGE_NAME} - docker save "${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz + docker save "${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz - name: Stop and remove docker containers run: | @@ -171,7 +171,8 @@ jobs: - name: Load previous version docker image if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} run: | - docker load --input druid-container-jdk8-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz + docker load --input druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz + docker images - name: Run IT id: run-it diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 7a8251e0afd8..c3b03d6455f1 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -124,9 +124,9 @@ jobs: - name: Save previous version docker image if: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} run: | - docker tag ${DRUID_PREVIOUS_IT_IMAGE_NAME} ${DRUID_PREVIOUS_IT_IMAGE_NAME}-jdk${{ inputs.build_jdk }}-version${DRUID_PREVIOUS_VERSION} + docker tag ${DRUID_PREVIOUS_IT_IMAGE_NAME} ${DRUID_PREVIOUS_IT_IMAGE_NAME}-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION} echo ${DRUID_PREVIOUS_IT_IMAGE_NAME} - docker save "${DRUID_PREVIOUS_IT_IMAGE_NAME}" | gzip > druid-container-jdk${{ inputs.build_jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz + docker save "${DRUID_PREVIOUS_IT_IMAGE_NAME}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz unit-tests-phase2: strategy: From f977f2440f26815bd8d40320f0ea3b54ea90a06a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 14:51:59 +0530 Subject: [PATCH 17/28] cache the previous image as well --- .github/workflows/unit-and-integration-tests-unified.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index c3b03d6455f1..00dba14c240a 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -99,6 +99,14 @@ jobs: ./druid-container-jdk${{ matrix.jdk }}.tar.gz ./integration-tests-ex/image/target/env.sh + - name: Cache previous version image + id: docker_container_previous_version + uses: actions/cache@v4 + with: + key: druid-container-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz-${{ github.sha }} + path: | + ./druid-container-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz + - name: Maven build id: maven_build run: | From 3a5c798c768e661340be88bd7c810d8bd04b30b9 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 14:56:43 +0530 Subject: [PATCH 18/28] minor change --- .github/workflows/unit-and-integration-tests-unified.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 00dba14c240a..900a71d28d75 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -103,9 +103,9 @@ jobs: id: docker_container_previous_version uses: actions/cache@v4 with: - key: druid-container-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz-${{ github.sha }} + key: druid-container-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }} path: | - ./druid-container-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz + ./druid-container-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }}.tar.gz - name: Maven build id: maven_build From 8811712dea4fc006627abb788b491cf3f7b2bd65 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 14:59:36 +0530 Subject: [PATCH 19/28] fix env var usage --- .github/workflows/unit-and-integration-tests-unified.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 900a71d28d75..db07902feaa9 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -132,9 +132,9 @@ jobs: - name: Save previous version docker image if: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} run: | - docker tag ${DRUID_PREVIOUS_IT_IMAGE_NAME} ${DRUID_PREVIOUS_IT_IMAGE_NAME}-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION} - echo ${DRUID_PREVIOUS_IT_IMAGE_NAME} - docker save "${DRUID_PREVIOUS_IT_IMAGE_NAME}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${DRUID_PREVIOUS_VERSION}.tar.gz + docker tag ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }} + echo ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} + docker save "${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }}.tar.gz unit-tests-phase2: strategy: From f8aa1648594bc01687037d4764fecd7f064664da Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 16:16:04 +0530 Subject: [PATCH 20/28] Remove test changes to set env var in it.sh --- it.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/it.sh b/it.sh index c8d3a74f8357..e5bb35de62c8 100755 --- a/it.sh +++ b/it.sh @@ -242,10 +242,10 @@ case $CMD in ;; "image" ) echo "it.sh printing env vars" - export DRUID_PREVIOUS_VERSION=30.0.0 - export BACKWARD_INCOMPATIBILITY_IT_ENABLED=true - export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz - export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:30.0.0 + #export DRUID_PREVIOUS_VERSION=30.0.0 + #export BACKWARD_INCOMPATIBILITY_IT_ENABLED=true + #export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz + #export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:30.0.0 echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED echo $DRUID_PREVIOUS_VERSION echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL From b067b301bf435480e9f7f7bf9405957bced57595 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 17:28:54 +0530 Subject: [PATCH 21/28] Debug failure in untarring the previous version --- integration-tests-ex/image/docker-build.sh | 6 ++++++ integration-tests-ex/image/docker/test-setup.sh | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index d5351a3aa2e0..5a4ba4ce866b 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -62,9 +62,15 @@ if [[ -z "${BACKWARD_INCOMPATIBILITY_IT_ENABLED:-""}" || $BACKWARD_INCOMPATIBILI exit 0 fi +echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL +echo $DRUID_PREVIOUS_VERSION + # Download the previous druid tar curl -L $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL --output apache-druid-$DRUID_PREVIOUS_VERSION-bin.tar.gz +echo "untarring " +tar -xzf apache-druid-$DRUID_PREVIOUS_VERSION-bin.tar.gz + docker build -t $DRUID_PREVIOUS_IT_IMAGE_NAME \ --build-arg DRUID_VERSION=$DRUID_PREVIOUS_VERSION \ --build-arg MYSQL_VERSION=$MYSQL_VERSION \ diff --git a/integration-tests-ex/image/docker/test-setup.sh b/integration-tests-ex/image/docker/test-setup.sh index 96488ad58531..9e7b0d6849da 100644 --- a/integration-tests-ex/image/docker/test-setup.sh +++ b/integration-tests-ex/image/docker/test-setup.sh @@ -24,10 +24,10 @@ set -e set -u # Enable for tracing -#set -x +set -x # For debugging: verify environment -#env +env # Druid system user adduser --system --group --no-create-home druid @@ -58,6 +58,7 @@ chmod a+x /run-druid.sh # should be added in this step. cd /usr/local/ +echo "untar the zip" tar -xzf apache-druid-${DRUID_VERSION}-bin.tar.gz rm apache-druid-${DRUID_VERSION}-bin.tar.gz From 721532c051ba02e72363f3de43c9de5b59ef4342 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 22 Jul 2024 21:04:24 +0530 Subject: [PATCH 22/28] Cleanup test changes --- .github/workflows/codeql.yml | 33 ++++++++---------- .github/workflows/distribution-checks.yml | 34 +++++++++---------- .github/workflows/revised-its.yml | 8 ----- .github/workflows/static-checks.yml | 22 ++++++------ .../unit-and-integration-tests-unified.yml | 14 +++----- integration-tests-ex/image/docker-build.sh | 7 ---- integration-tests-ex/image/docker/Dockerfile | 1 - .../image/docker/test-setup.sh | 5 ++- integration-tests-ex/image/pom.xml | 8 ----- it.sh | 16 --------- 10 files changed, 48 insertions(+), 100 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e4cd90de85e5..7cfeb63cbff0 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,24 +1,21 @@ name: "CodeQL" on: - workflow_call: - -#on: -# push: -# paths-ignore: -# - '**/*.md' -# - 'dev/**' -# - 'docs/**' -# branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ] -# pull_request: -# paths-ignore: -# - '**/*.md' -# - 'dev/**' -# - 'docs/**' -# # The branches below must be a subset of the branches above -# branches: [ 'master' ] -# schedule: -# - cron: '18 15 * * 4' + push: + paths-ignore: + - '**/*.md' + - 'dev/**' + - 'docs/**' + branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ] + pull_request: + paths-ignore: + - '**/*.md' + - 'dev/**' + - 'docs/**' + # The branches below must be a subset of the branches above + branches: [ 'master' ] + schedule: + - cron: '18 15 * * 4' jobs: analyze: diff --git a/.github/workflows/distribution-checks.yml b/.github/workflows/distribution-checks.yml index feb188eb3d04..caadb16b608e 100644 --- a/.github/workflows/distribution-checks.yml +++ b/.github/workflows/distribution-checks.yml @@ -15,24 +15,22 @@ name: "Distribution Checks" on: - workflow_call: - -# push: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches -# paths: -# - 'distribution/**' -# - '**/pom.xml' -# pull_request: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches -# paths: -# - 'distribution/**' -# - '**/pom.xml' + push: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + paths: + - 'distribution/**' + - '**/pom.xml' + pull_request: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + paths: + - 'distribution/**' + - '**/pom.xml' jobs: docker-build: diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 1cd144a55135..c79a6479f6d6 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -80,10 +80,6 @@ jobs: script: ./it.sh github ${{ matrix.it }} it: ${{ matrix.it }} mysql_driver: com.mysql.jdbc.Driver - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} s3-deep-storage-minio: needs: changes @@ -101,10 +97,6 @@ jobs: AWS_REGION: us-east-1 AWS_ACCESS_KEY_ID: admin AWS_SECRET_ACCESS_KEY: miniopassword - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} backward-compatibility-it: needs: changes diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 82bef910cde8..a374cf72ccfe 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -15,18 +15,16 @@ name: "Static Checks CI" on: - workflow_call: - -# push: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches -# pull_request: -# branches: -# - master -# - '[0-9]+.[0-9]+.[0-9]+' # release branches -# - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + push: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches + pull_request: + branches: + - master + - '[0-9]+.[0-9]+.[0-9]+' # release branches + - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.run_id }}' # group workflows only on pull_requests and not on branch commits diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index db07902feaa9..b05052f0a81c 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -58,8 +58,7 @@ jobs: strategy: fail-fast: false matrix: - #jdk: [ '8', '11', '17', '21' ] - jdK: [ '8' ] + jdk: [ '8', '11', '17', '21' ] runs-on: ubuntu-latest steps: - name: Checkout branch @@ -144,13 +143,12 @@ jobs: name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests - if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} with: jdk: ${{ matrix.jdk }} sql_compatibility: true unit-tests: - if: false strategy: fail-fast: false matrix: @@ -164,12 +162,10 @@ jobs: standard-its: needs: unit-tests - if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/standard-its.yml expose-vars: - #needs: unit-tests - needs: build runs-on: ubuntu-latest outputs: BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} @@ -180,8 +176,8 @@ jobs: - run: echo "Exposing env vars" revised-its: - needs: expose-vars - if: true #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + needs: [unit-tests, expose-vars] + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml with: BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ needs.expose-vars.outputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index 5a4ba4ce866b..8afee10e440b 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -56,21 +56,14 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . -echo "val is $BACKWARD_INCOMPATIBILITY_IT_ENABLED" if [[ -z "${BACKWARD_INCOMPATIBILITY_IT_ENABLED:-""}" || $BACKWARD_INCOMPATIBILITY_IT_ENABLED != "true" ]]; then echo "Not building previous version image." exit 0 fi -echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL -echo $DRUID_PREVIOUS_VERSION - # Download the previous druid tar curl -L $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL --output apache-druid-$DRUID_PREVIOUS_VERSION-bin.tar.gz -echo "untarring " -tar -xzf apache-druid-$DRUID_PREVIOUS_VERSION-bin.tar.gz - docker build -t $DRUID_PREVIOUS_IT_IMAGE_NAME \ --build-arg DRUID_VERSION=$DRUID_PREVIOUS_VERSION \ --build-arg MYSQL_VERSION=$MYSQL_VERSION \ diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index 99b2da9476b2..d76509ee406f 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -73,4 +73,3 @@ RUN bash /test-setup.sh USER druid:druid WORKDIR / ENTRYPOINT [ "bash", "/launch.sh" ] - diff --git a/integration-tests-ex/image/docker/test-setup.sh b/integration-tests-ex/image/docker/test-setup.sh index 9e7b0d6849da..96488ad58531 100644 --- a/integration-tests-ex/image/docker/test-setup.sh +++ b/integration-tests-ex/image/docker/test-setup.sh @@ -24,10 +24,10 @@ set -e set -u # Enable for tracing -set -x +#set -x # For debugging: verify environment -env +#env # Druid system user adduser --system --group --no-create-home druid @@ -58,7 +58,6 @@ chmod a+x /run-druid.sh # should be added in this step. cd /usr/local/ -echo "untar the zip" tar -xzf apache-druid-${DRUID_VERSION}-bin.tar.gz rm apache-druid-${DRUID_VERSION}-bin.tar.gz diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index 6467152819c4..600501edb92b 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -55,10 +55,6 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven image name will typically be provided by the build enfironment, and will override this default name. --> ${project.groupId}/test:${project.version} - 5.5.1 2.7.3 5.7-debian @@ -213,10 +209,6 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven ${hadoop.compile.version} ${project.version} ${druid.it.image-name} - ${project.build.directory} diff --git a/it.sh b/it.sh index e5bb35de62c8..12519c24b54d 100755 --- a/it.sh +++ b/it.sh @@ -241,15 +241,6 @@ case $CMD in mvn -B install -pl :druid-it-tools ;; "image" ) - echo "it.sh printing env vars" - #export DRUID_PREVIOUS_VERSION=30.0.0 - #export BACKWARD_INCOMPATIBILITY_IT_ENABLED=true - #export DRUID_PREVIOUS_VERSION_DOWNLOAD_URL=https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz - #export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:30.0.0 - echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED - echo $DRUID_PREVIOUS_VERSION - echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL - echo $DRUID_PREVIOUS_IT_IMAGE_NAME cd $DRUID_DEV/integration-tests-ex/image mvn -B install -P test-image $MAVEN_IGNORE ;; @@ -295,17 +286,10 @@ case $CMD in tail_logs ;; "github" ) - echo "it.sh printing env vars" - echo $BACKWARD_INCOMPATIBILITY_IT_ENABLED - echo $DRUID_PREVIOUS_VERSION - echo $DRUID_PREVIOUS_VERSION_DOWNLOAD_URL - echo $DRUID_PREVIOUS_IT_IMAGE_NAME - set +e $0 test $CATEGORY RESULT=$? - # Include logs, but only for failures. if [ $RESULT -ne 0 ]; then $0 tail $CATEGORY From 36044512bbe786bfe4a19122961afd37c4b5aa8c Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 26 Jul 2024 16:09:28 +0530 Subject: [PATCH 23/28] Derive BACKWARD_INCOMPATIBILITY_IT_ENABLED and DRUID_PREVIOUS_IT_IMAGE_NAME env vars --- .../unit-and-integration-tests-unified.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index b05052f0a81c..f7c5d9b9b64d 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -47,13 +47,28 @@ concurrency: env: MYSQL_DRIVER_CLASSNAME: com.mysql.jdbc.Driver # Used to set druid config in docker image for revised ITs SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 - BACKWARD_INCOMPATIBILITY_IT_ENABLED: true + #BACKWARD_INCOMPATIBILITY_IT_ENABLED: true DRUID_PREVIOUS_VERSION: 30.0.0 DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz - DRUID_PREVIOUS_IT_IMAGE_NAME: org.apache.druid.integration-tests/test:30.0.0 + #DRUID_PREVIOUS_IT_IMAGE_NAME: org.apache.druid.integration-tests/test:30.0.0 jobs: + set-new-env-var: + name: Set new env var + runs-on: ubuntu-latest + steps: + - name: Set new environment variable + run: | + if [ -n "${{ env.DRUID_PREVIOUS_VERSION }}" ]; then + echo "export BACKWARD_INCOMPATIBILITY_IT_ENABLED=true" >> $GITHUB_ENV + else + echo "export BACKWARD_INCOMPATIBILITY_IT_ENABLED=false" >> $GITHUB_ENV + fi + echo "export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:${{ env.DRUID_PREVIOUS_VERSION }}" >> $GITHUB_ENV + echo "New environment variable: $DRUID_PREVIOUS_IT_IMAGE_NAME, $BACKWARD_INCOMPATIBILITY_IT_ENABLED" + build: + needs: set-new-env-var name: "build (jdk${{ matrix.jdk }})" strategy: fail-fast: false From 3ac238510b8d3193d9a1e9510157547b451b85d4 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 26 Jul 2024 18:18:01 +0530 Subject: [PATCH 24/28] Rename test group to BackwardCompatibility --- .../docker-compose.yaml | 0 .../cases/cluster/Common/dependencies.yaml | 6 +++--- integration-tests-ex/cases/pom.xml | 4 ++-- .../ITBackwardCompatibilityIndexerTest.java} | 8 ++++---- ...ibility.java => BackwardCompatibility.java} | 2 +- .../docker.yaml | 0 integration-tests-ex/image/build-image.sh | 1 + integration-tests-ex/image/docker-build.sh | 5 ++--- integration-tests-ex/image/docker/Dockerfile | 18 ++++++++++++++---- 9 files changed, 27 insertions(+), 17 deletions(-) rename integration-tests-ex/cases/cluster/{BackwardIncompatibility => BackwardCompatibility}/docker-compose.yaml (100%) rename integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/{BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java => BackwardCompatibility/ITBackwardCompatibilityIndexerTest.java} (81%) rename integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/{BackwardIncompatibility.java => BackwardCompatibility.java} (95%) rename integration-tests-ex/cases/src/test/resources/cluster/{BackwardIncompatibility => BackwardCompatibility}/docker.yaml (100%) diff --git a/integration-tests-ex/cases/cluster/BackwardIncompatibility/docker-compose.yaml b/integration-tests-ex/cases/cluster/BackwardCompatibility/docker-compose.yaml similarity index 100% rename from integration-tests-ex/cases/cluster/BackwardIncompatibility/docker-compose.yaml rename to integration-tests-ex/cases/cluster/BackwardCompatibility/docker-compose.yaml diff --git a/integration-tests-ex/cases/cluster/Common/dependencies.yaml b/integration-tests-ex/cases/cluster/Common/dependencies.yaml index 0adf603ff91e..0409c30bf538 100644 --- a/integration-tests-ex/cases/cluster/Common/dependencies.yaml +++ b/integration-tests-ex/cases/cluster/Common/dependencies.yaml @@ -26,7 +26,7 @@ services: # See https://hub.docker.com/_/zookeeper zookeeper: # Uncomment the following when running on Apple Silicon processors: - platform: linux/x86_64 + # platform: linux/x86_64 image: zookeeper:${ZK_VERSION} container_name: zookeeper labels: @@ -46,7 +46,7 @@ services: kafka: image: bitnami/kafka:${KAFKA_VERSION} container_name: kafka - platform: linux/x86_64 + # platform: linux/x86_64 labels: druid-int-test: "true" ports: @@ -73,7 +73,7 @@ services: # The image will intialize the user and DB upon first start. metadata: # Uncomment the following when running on Apple Silicon processors: - platform: linux/x86_64 + # platform: linux/x86_64 image: mysql:$MYSQL_IMAGE_VERSION container_name: metadata labels: diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 260a598a4733..a07dad2d14fe 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -460,12 +460,12 @@ - IT-BackwardIncompatibility + IT-BackwardCompatibility false - BackwardIncompatibility + BackwardCompatibility diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardCompatibility/ITBackwardCompatibilityIndexerTest.java similarity index 81% rename from integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java rename to integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardCompatibility/ITBackwardCompatibilityIndexerTest.java index e0578a3a922f..7d07d518922e 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardIncompatibility/ITBackwardIncompatibilityIndexerTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/BackwardCompatibility/ITBackwardCompatibilityIndexerTest.java @@ -17,16 +17,16 @@ * under the License. */ -package org.apache.druid.testsEx.BackwardIncompatibility; +package org.apache.druid.testsEx.BackwardCompatibility; -import org.apache.druid.testsEx.categories.BackwardIncompatibility; +import org.apache.druid.testsEx.categories.BackwardCompatibility; import org.apache.druid.testsEx.config.DruidTestRunner; import org.apache.druid.testsEx.indexer.IndexerTest; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @RunWith(DruidTestRunner.class) -@Category({BackwardIncompatibility.class}) -public class ITBackwardIncompatibilityIndexerTest extends IndexerTest +@Category({BackwardCompatibility.class}) +public class ITBackwardCompatibilityIndexerTest extends IndexerTest { } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardIncompatibility.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardCompatibility.java similarity index 95% rename from integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardIncompatibility.java rename to integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardCompatibility.java index 945cc3bedac0..7e357e6df8f0 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardIncompatibility.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/BackwardCompatibility.java @@ -19,6 +19,6 @@ package org.apache.druid.testsEx.categories; -public class BackwardIncompatibility +public class BackwardCompatibility { } diff --git a/integration-tests-ex/cases/src/test/resources/cluster/BackwardIncompatibility/docker.yaml b/integration-tests-ex/cases/src/test/resources/cluster/BackwardCompatibility/docker.yaml similarity index 100% rename from integration-tests-ex/cases/src/test/resources/cluster/BackwardIncompatibility/docker.yaml rename to integration-tests-ex/cases/src/test/resources/cluster/BackwardCompatibility/docker.yaml diff --git a/integration-tests-ex/image/build-image.sh b/integration-tests-ex/image/build-image.sh index 65bd816b1248..4a31a7841944 100755 --- a/integration-tests-ex/image/build-image.sh +++ b/integration-tests-ex/image/build-image.sh @@ -32,6 +32,7 @@ SCRIPT_DIR=$(cd $(dirname $0) && pwd) # Copy environment variables to a file. Used for manual rebuilds # and by scripts that start the test cluster. + cat > $TARGET_DIR/env.sh << EOF export ZK_VERSION=$ZK_VERSION export KAFKA_VERSION=$KAFKA_VERSION diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index 8afee10e440b..991fa5a3fff4 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -53,10 +53,10 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg CONFLUENT_VERSION=$CONFLUENT_VERSION \ --build-arg HADOOP_VERSION=$HADOOP_VERSION \ --build-arg MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME \ - --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ + --build-arg COPY_DRUID_TESTING_TOOLS=1 \ . -if [[ -z "${BACKWARD_INCOMPATIBILITY_IT_ENABLED:-""}" || $BACKWARD_INCOMPATIBILITY_IT_ENABLED != "true" ]]; then +if [[ -z "${BACKWARD_COMPATIBILITY_IT_ENABLED:-""}" || $BACKWARD_COMPATIBILITY_IT_ENABLED != "true" ]]; then echo "Not building previous version image." exit 0 fi @@ -71,5 +71,4 @@ docker build -t $DRUID_PREVIOUS_IT_IMAGE_NAME \ --build-arg CONFLUENT_VERSION=$CONFLUENT_VERSION \ --build-arg HADOOP_VERSION=$HADOOP_VERSION \ --build-arg MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME \ - --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index d76509ee406f..a6e5f07603c0 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -34,7 +34,16 @@ ARG JDK_VERSION=8-slim-buster # thus provides bash, apt-get, etc. # See https://hub.docker.com/_/openjdk -FROM openjdk:$JDK_VERSION +FROM openjdk:$JDK_VERSION as base + +ARG DRUID_VERSION +ENV DRUID_VERSION=$DRUID_VERSION +ARG COPY_DRUID_TESTING_TOOLS + +FROM base as with-druid-testing-tools +COPY druid-it-tools-${DRUID_VERSION}.jar /tmp/druid/extensions/druid-it-tools/ + +FROM base as final ARG DRUID_VERSION ENV DRUID_VERSION=$DRUID_VERSION @@ -46,15 +55,15 @@ ARG MARIADB_VERSION ENV MARIADB_VERSION=$MARIADB_VERSION ARG MYSQL_DRIVER_CLASSNAME=com.mysql.jdbc.Driver ENV MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME -ARG DRUID_TESTING_TOOLS_VERSION -ENV DRUID_TESTING_TOOLS_VERSION=$DRUID_TESTING_TOOLS_VERSION +ARG COPY_DRUID_TESTING_TOOLS ENV DRUID_HOME=/usr/local/druid # Populate build artifacts COPY apache-druid-${DRUID_VERSION}-bin.tar.gz /usr/local/ -COPY druid-it-tools-${DRUID_TESTING_TOOLS_VERSION}.jar /tmp/druid/extensions/druid-it-tools/ +COPY --from=with-druid-testing-tools /tmp/druid/extensions/druid-it-tools/ /tmp/druid/extensions/druid-it-tools/ + COPY kafka-protobuf-provider-${CONFLUENT_VERSION}.jar /tmp/druid/lib/ COPY mysql-connector-j-${MYSQL_VERSION}.jar /tmp/druid/lib/ COPY mariadb-java-client-${MARIADB_VERSION}.jar /tmp/druid/lib/ @@ -62,6 +71,7 @@ COPY test-setup.sh / COPY druid.sh / COPY launch.sh / + # Do the setup tasks. The tasks are done within a script, rather than # here, so they are easier to describe and debug. Turn on the "-x" flag # within the script to trace the steps if needed for debugging. From 7c34af318200a7b37bc6164829a7d1719fcbc648 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 26 Jul 2024 20:24:30 +0530 Subject: [PATCH 25/28] rename config var in gha --- .github/workflows/reusable-revised-its.yml | 12 ++++++------ .github/workflows/revised-its.yml | 6 +++--- .../unit-and-integration-tests-unified.yml | 18 +++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index 254536e66944..1aa29f64cb5b 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -57,7 +57,7 @@ on: AWS_SECRET_ACCESS_KEY: required: false type: string - BACKWARD_INCOMPATIBILITY_IT_ENABLED: + BACKWARD_COMPATIBILITY_IT_ENABLED: required: false type: string default: false @@ -121,7 +121,7 @@ jobs: - name: Retrieve previous version cached docker image id: docker-restore-previous-version - if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} + if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }} uses: actions/cache/restore@v4 with: key: druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }} @@ -137,7 +137,7 @@ jobs: if: steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true' env: docker-restore: ${{ toJson(steps.docker-restore.outputs) }} - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} @@ -149,7 +149,7 @@ jobs: docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ inputs.build_jdk }}.tar.gz - name: Save previous version docker image - if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' && (steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true') }} + if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true') }} env: docker-restore: ${{ toJson(steps.docker-restore.outputs) }} run: | @@ -169,7 +169,7 @@ jobs: docker images - name: Load previous version docker image - if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} + if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }} run: | docker load --input druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz docker images @@ -177,7 +177,7 @@ jobs: - name: Run IT id: run-it env: - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index c79a6479f6d6..0acb3a2daec5 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -19,7 +19,7 @@ name: "Revised ITs workflow" on: workflow_call: inputs: - BACKWARD_INCOMPATIBILITY_IT_ENABLED: + BACKWARD_COMPATIBILITY_IT_ENABLED: description: "Flag for backward compatibility IT" required: false default: false @@ -101,7 +101,7 @@ jobs: backward-compatibility-it: needs: changes uses: ./.github/workflows/reusable-revised-its.yml - if: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }} + if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }} with: build_jdk: 8 runtime_jdk: 8 @@ -109,7 +109,7 @@ jobs: script: ./it.sh github BackwardIncompatibility it: BackwardIncompatibility mysql_driver: com.mysql.jdbc.Driver - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }} DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index f7c5d9b9b64d..355570a31217 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -47,9 +47,9 @@ concurrency: env: MYSQL_DRIVER_CLASSNAME: com.mysql.jdbc.Driver # Used to set druid config in docker image for revised ITs SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 - #BACKWARD_INCOMPATIBILITY_IT_ENABLED: true DRUID_PREVIOUS_VERSION: 30.0.0 DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: https://dlcdn.apache.org/druid/30.0.0/apache-druid-30.0.0-bin.tar.gz + #BACKWARD_COMPATIBILITY_IT_ENABLED: true #DRUID_PREVIOUS_IT_IMAGE_NAME: org.apache.druid.integration-tests/test:30.0.0 jobs: @@ -60,12 +60,12 @@ jobs: - name: Set new environment variable run: | if [ -n "${{ env.DRUID_PREVIOUS_VERSION }}" ]; then - echo "export BACKWARD_INCOMPATIBILITY_IT_ENABLED=true" >> $GITHUB_ENV + echo "export BACKWARD_COMPATIBILITY_IT_ENABLED=true" >> $GITHUB_ENV else - echo "export BACKWARD_INCOMPATIBILITY_IT_ENABLED=false" >> $GITHUB_ENV + echo "export BACKWARD_COMPATIBILITY_IT_ENABLED=false" >> $GITHUB_ENV fi echo "export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:${{ env.DRUID_PREVIOUS_VERSION }}" >> $GITHUB_ENV - echo "New environment variable: $DRUID_PREVIOUS_IT_IMAGE_NAME, $BACKWARD_INCOMPATIBILITY_IT_ENABLED" + echo "New environment variable: $DRUID_PREVIOUS_IT_IMAGE_NAME, $BACKWARD_COMPATIBILITY_IT_ENABLED" build: needs: set-new-env-var @@ -128,7 +128,7 @@ jobs: - name: Container build env: - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_COMPATIBILITY_IT_ENABLED }} DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} @@ -144,7 +144,7 @@ jobs: docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz - name: Save previous version docker image - if: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED == 'true' }} + if: ${{ env.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }} run: | docker tag ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }} echo ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} @@ -183,10 +183,10 @@ jobs: expose-vars: runs-on: ubuntu-latest outputs: - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ BACKWARD_COMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ DRUID_PREVIOUS_IT_IMAGE_NAME }} steps: - run: echo "Exposing env vars" @@ -195,7 +195,7 @@ jobs: if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml with: - BACKWARD_INCOMPATIBILITY_IT_ENABLED: ${{ needs.expose-vars.outputs.BACKWARD_INCOMPATIBILITY_IT_ENABLED }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.expose-vars.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} DRUID_PREVIOUS_VERSION: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_VERSION }} DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} From 5054ba5eac51a75c0cd0520864fe50b8bb8fe9ed Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 26 Jul 2024 20:56:49 +0530 Subject: [PATCH 26/28] Update job to set env var --- .../unit-and-integration-tests-unified.yml | 72 ++++++++++--------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 355570a31217..c27c7da71291 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -53,22 +53,30 @@ env: #DRUID_PREVIOUS_IT_IMAGE_NAME: org.apache.druid.integration-tests/test:30.0.0 jobs: - set-new-env-var: - name: Set new env var + set-env-var: + name: Set env var runs-on: ubuntu-latest + outputs: + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ steps.image_name.outputs.image_name }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ steps.it_enabled.outputs.enabled }} + DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} steps: - - name: Set new environment variable + - name: Set image name env var + id: image_name + run: | + echo "::set-output name=image_name::org.apache.druid.integration-tests/test:${{ env.DRUID_PREVIOUS_VERSION }}" + - name: Set env for enabling backward compatibility it + id: it_enabled run: | if [ -n "${{ env.DRUID_PREVIOUS_VERSION }}" ]; then - echo "export BACKWARD_COMPATIBILITY_IT_ENABLED=true" >> $GITHUB_ENV + echo "::set-output name=enabled::true" else - echo "export BACKWARD_COMPATIBILITY_IT_ENABLED=false" >> $GITHUB_ENV + echo "::set-output name=enabled::false" fi - echo "export DRUID_PREVIOUS_IT_IMAGE_NAME=org.apache.druid.integration-tests/test:${{ env.DRUID_PREVIOUS_VERSION }}" >> $GITHUB_ENV - echo "New environment variable: $DRUID_PREVIOUS_IT_IMAGE_NAME, $BACKWARD_COMPATIBILITY_IT_ENABLED" build: - needs: set-new-env-var + needs: set-env-var name: "build (jdk${{ matrix.jdk }})" strategy: fail-fast: false @@ -117,9 +125,9 @@ jobs: id: docker_container_previous_version uses: actions/cache@v4 with: - key: druid-container-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }} + key: druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }} path: | - ./druid-container-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }}.tar.gz + ./druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz - name: Maven build id: maven_build @@ -128,10 +136,10 @@ jobs: - name: Container build env: - BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ env.BACKWARD_COMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} run: | ./it.sh image source ./integration-tests-ex/image/target/env.sh @@ -144,11 +152,11 @@ jobs: docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz - name: Save previous version docker image - if: ${{ env.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }} + if: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }} run: | - docker tag ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }} - echo ${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }} - docker save "${{ env.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${{ env.DRUID_PREVIOUS_VERSION }}.tar.gz + docker tag ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }} + echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} + docker save "${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz unit-tests-phase2: strategy: @@ -180,22 +188,22 @@ jobs: if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/standard-its.yml - expose-vars: - runs-on: ubuntu-latest - outputs: - DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ BACKWARD_COMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ DRUID_PREVIOUS_IT_IMAGE_NAME }} - steps: - - run: echo "Exposing env vars" +# expose-vars: +# runs-on: ubuntu-latest +# outputs: +# DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }} +# DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} +# BACKWARD_COMPATIBILITY_IT_ENABLED: $BACKWARD_COMPATIBILITY_IT_ENABLED +# DRUID_PREVIOUS_IT_IMAGE_NAME: $DRUID_PREVIOUS_IT_IMAGE_NAME +# steps: +# - run: echo "Exposing env vars" revised-its: - needs: [unit-tests, expose-vars] + needs: [unit-tests, set-env-var] if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml with: - BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.expose-vars.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} - DRUID_PREVIOUS_VERSION: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_VERSION }} - DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} - DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.expose-vars.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} + BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} + DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }} + DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} From 7aaa9fd4bf3025a36b7d00345893be5f7bf35de8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 26 Jul 2024 21:00:57 +0530 Subject: [PATCH 27/28] temp step to debug env vars are set correctly --- .../workflows/unit-and-integration-tests-unified.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index c27c7da71291..f3d9cb40555f 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -198,6 +198,18 @@ jobs: # steps: # - run: echo "Exposing env vars" + print-env-vars: + needs: set-env-var + runs-on: ubuntu-latest + steps: + - name: print env vars + id: test + run: | + echo ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }} + echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }} + echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }} + echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} + revised-its: needs: [unit-tests, set-env-var] if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} From bf93720d28f0064a646847ca09b70ccb094f0949 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 26 Jul 2024 22:48:21 +0530 Subject: [PATCH 28/28] minor change --- integration-tests-ex/image/docker-build.sh | 3 ++- integration-tests-ex/image/docker/Dockerfile | 15 +++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/integration-tests-ex/image/docker-build.sh b/integration-tests-ex/image/docker-build.sh index 991fa5a3fff4..4ff5d7a74387 100755 --- a/integration-tests-ex/image/docker-build.sh +++ b/integration-tests-ex/image/docker-build.sh @@ -53,7 +53,7 @@ docker build -t $DRUID_IT_IMAGE_NAME \ --build-arg CONFLUENT_VERSION=$CONFLUENT_VERSION \ --build-arg HADOOP_VERSION=$HADOOP_VERSION \ --build-arg MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME \ - --build-arg COPY_DRUID_TESTING_TOOLS=1 \ + --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . if [[ -z "${BACKWARD_COMPATIBILITY_IT_ENABLED:-""}" || $BACKWARD_COMPATIBILITY_IT_ENABLED != "true" ]]; then @@ -71,4 +71,5 @@ docker build -t $DRUID_PREVIOUS_IT_IMAGE_NAME \ --build-arg CONFLUENT_VERSION=$CONFLUENT_VERSION \ --build-arg HADOOP_VERSION=$HADOOP_VERSION \ --build-arg MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME \ + --build-arg DRUID_TESTING_TOOLS_VERSION=$DRUID_VERSION \ . diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index a6e5f07603c0..90955eae3c11 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -34,16 +34,7 @@ ARG JDK_VERSION=8-slim-buster # thus provides bash, apt-get, etc. # See https://hub.docker.com/_/openjdk -FROM openjdk:$JDK_VERSION as base - -ARG DRUID_VERSION -ENV DRUID_VERSION=$DRUID_VERSION -ARG COPY_DRUID_TESTING_TOOLS - -FROM base as with-druid-testing-tools -COPY druid-it-tools-${DRUID_VERSION}.jar /tmp/druid/extensions/druid-it-tools/ - -FROM base as final +FROM openjdk:$JDK_VERSION ARG DRUID_VERSION ENV DRUID_VERSION=$DRUID_VERSION @@ -55,14 +46,14 @@ ARG MARIADB_VERSION ENV MARIADB_VERSION=$MARIADB_VERSION ARG MYSQL_DRIVER_CLASSNAME=com.mysql.jdbc.Driver ENV MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME -ARG COPY_DRUID_TESTING_TOOLS +ARG DRUID_TESTING_TOOLS_VERSION ENV DRUID_HOME=/usr/local/druid # Populate build artifacts COPY apache-druid-${DRUID_VERSION}-bin.tar.gz /usr/local/ -COPY --from=with-druid-testing-tools /tmp/druid/extensions/druid-it-tools/ /tmp/druid/extensions/druid-it-tools/ +COPY druid-it-tools-${DRUID_TESTING_TOOLS_VERSION}.jar /tmp/druid/extensions/druid-it-tools/ COPY kafka-protobuf-provider-${CONFLUENT_VERSION}.jar /tmp/druid/lib/ COPY mysql-connector-j-${MYSQL_VERSION}.jar /tmp/druid/lib/