diff --git a/eng/pipelines/scripts/generate-project-list.ps1 b/eng/pipelines/scripts/generate-project-list.ps1 index a4432a8b3977..60e00fe65af4 100644 --- a/eng/pipelines/scripts/generate-project-list.ps1 +++ b/eng/pipelines/scripts/generate-project-list.ps1 @@ -4,6 +4,22 @@ $additionalModulesList = @() . "${PSScriptRoot}/../../common/scripts/common.ps1" +# If ProjectListOverride is set (e.g., from matrix variables), use it directly +# to avoid building unnecessary modules in jobs that only test a subset. +if ($env:PROJECTLISTOVERRIDE -and $env:PROJECTLISTOVERRIDE -notlike '*ProjectListOverride*') { + $projects = $env:PROJECTLISTOVERRIDE + Write-Host "Using ProjectListOverride = $projects" + Write-Host "##vso[task.setvariable variable=ProjectList;]$projects" + Write-Host "##vso[task.setvariable variable=ArtifactsList;]$projects" + Write-Host "##vso[task.setvariable variable=AdditionalModulesList;]" + + $sha256 = new-object -TypeName System.Security.Cryptography.SHA256Managed + $utf8 = new-object -TypeName System.Text.UTF8Encoding + $projectListSha256 = [Convert]::ToBase64String($sha256.ComputeHash($utf8.GetBytes($projects))) + Write-Host "##vso[task.setvariable variable=ProjectListSha256;]$projectListSha256" + return +} + if ($env:ARTIFACTSJSON -and $env:ARTIFACTSJSON -notlike '*ArtifactsJson*') { $artifacts = $env:ARTIFACTSJSON | ConvertFrom-Json foreach ($artifact in $artifacts) { diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 82d12ff45fdf..cdfac6273e60 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -40,6 +40,9 @@ parameters: - name: BuildParallelization type: string default: '2C' + - name: BuildOptions + type: string + default: '' - name: AdditionalLintingOptions type: string default: '' @@ -145,6 +148,7 @@ jobs: TestOptions: ${{ parameters.TestOptions }} TestParallelization: ${{ parameters.TestParallelization }} BuildParallelization: ${{ parameters.BuildParallelization }} + BuildOptions: ${{ parameters.BuildOptions }} TestEnvVars: ${{ parameters.EnvVars }} VersionOverride: ${{ parameters.VersionOverride }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 703c81308c01..95233db735c1 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -40,6 +40,9 @@ parameters: - name: BuildParallelization type: string default: '2C' + - name: BuildOptions + type: string + default: '' - name: TestGoals type: string default: $(TestGoals) @@ -503,6 +506,7 @@ jobs: TestOptions: ${{ parameters.TestOptions }} TestParallelization: ${{ parameters.TestParallelization }} BuildParallelization: ${{ parameters.BuildParallelization }} + BuildOptions: ${{ parameters.BuildOptions }} AdditionalLintingOptions: ${{ parameters.AdditionalLintingOptions }} TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }} EnvVars: ${{ parameters.EnvVars }} diff --git a/eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json b/eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json new file mode 100644 index 000000000000..a85b0344ab14 --- /dev/null +++ b/eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json @@ -0,0 +1,110 @@ +{ + "displayNames": { + "clean verify": "" + }, + "matrix": { + "Agent": { + "windows-2022": { + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL" + } + }, + "TestGoals": "clean verify", + "EmulatorConfig": { + "Emulator Only Integration Tests Tcp - Java 8": { + "ProfileFlag": "-Pemulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" + }, + "Emulator Only Integration Tests Tcp - Java 17": { + "ProfileFlag": "-Pemulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Strong\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" + }, + "Long Emulator Only Integration Tests Tcp - Java 8": { + "ProfileFlag": "-Plong-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" + }, + "Long Emulator Only Integration Tests Tcp - Java 17": { + "ProfileFlag": "-Plong-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Strong\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" + }, + "Encryption Emulator Only Integration Tests": { + "ProfileFlag": "-Pencryption-integration", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" + }, + "Spark 3.3 Integration Tests targeting Cosmos Emulator - Java 8'": { + "ProfileFlag": "-Dspark-e2e_3-3", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12" + }, + "Spark 3.4 Integration Tests targeting Cosmos Emulator - Java 11'": { + "ProfileFlag": "-Dspark-e2e_3-4", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.11", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-4_2-12" + }, + "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 17'": { + "ProfileFlag": "-Dspark-e2e_3-5_2-12", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-12" + }, + "Spark 3.5, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": { + "ProfileFlag": "-Dspark-e2e_3-5_2-13", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-13" + }, + "Spark 4.0, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 21'": { + "ProfileFlag": "-Dspark-e2e_4-0_2-13", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.21", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13" + }, + "Kafka Integration Tests targeting Cosmos Emulator - Java 17": { + "ProfileFlag": "-Pkafka-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.kafka:azure-cosmos-kafka-connect" + } + } + } +} diff --git a/eng/pipelines/templates/stages/cosmos-emulator-matrix.json b/eng/pipelines/templates/stages/cosmos-emulator-matrix.json index 2823975d5ea8..88dffb0886c8 100644 --- a/eng/pipelines/templates/stages/cosmos-emulator-matrix.json +++ b/eng/pipelines/templates/stages/cosmos-emulator-matrix.json @@ -4,121 +4,151 @@ }, "matrix": { "Agent": { - "windows-2022": { "OSVmImage": "env:WINDOWSVMIMAGE", "Pool": "env:WINDOWSPOOL" } + "windows-2022": { + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL" + } }, "TestGoals": "clean verify", "EmulatorConfig": { "Emulator Only Integration Tests Tcp - Java 8": { - "ProfileFlag": "-Pemulator", - "PROTOCOLS": "[\"Tcp\"]", - "DESIRED_CONSISTENCIES": "[\"Session\"]", - "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "ProfileFlag": "-Pemulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" }, "Emulator Only Integration Tests Tcp - Java 17": { "ProfileFlag": "-Pemulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Strong\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" }, "Long Emulator Only Integration Tests Tcp - Java 8": { - "ProfileFlag": "-Plong-emulator", - "PROTOCOLS": "[\"Tcp\"]", - "DESIRED_CONSISTENCIES": "[\"Session\"]", - "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "ProfileFlag": "-Plong-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" }, "Long Emulator Only Integration Tests Tcp - Java 17": { "ProfileFlag": "-Plong-emulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Strong\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" }, "Encryption Emulator Only Integration Tests": { - "ProfileFlag": "-Pencryption-integration", - "PROTOCOLS": "[\"Tcp\"]", - "DESIRED_CONSISTENCIES": "[\"Session\"]", - "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "ProfileFlag": "-Pencryption-integration", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true" }, "Spark 3.3 Integration Tests targeting Cosmos Emulator - Java 8'": { "ProfileFlag": "-Dspark-e2e_3-3", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12" }, "Spark 3.3 Integration Tests targeting Cosmos Emulator - Java 11'": { "ProfileFlag": "-Dspark-e2e_3-3", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.11", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12" }, "Spark 3.4 Integration Tests targeting Cosmos Emulator - Java 8'": { "ProfileFlag": "-Dspark-e2e_3-4", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-4_2-12" }, "Spark 3.4 Integration Tests targeting Cosmos Emulator - Java 11'": { "ProfileFlag": "-Dspark-e2e_3-4", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.11", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-4_2-12" }, "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 8'": { "ProfileFlag": "-Dspark-e2e_3-5_2-12", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-12" }, "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 17'": { "ProfileFlag": "-Dspark-e2e_3-5_2-12", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-12" }, "Spark 3.5, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": { "ProfileFlag": "-Dspark-e2e_3-5_2-13", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-13" }, "Spark 4.0, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": { "ProfileFlag": "-Dspark-e2e_4-0_2-13", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13" }, "Spark 4.0, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 21'": { "ProfileFlag": "-Dspark-e2e_4-0_2-13", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.21", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13" }, "Kafka Integration Tests targeting Cosmos Emulator - Java 11": { "ProfileFlag": "-Pkafka-emulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.11", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.kafka:azure-cosmos-kafka-connect" }, "Kafka Integration Tests targeting Cosmos Emulator - Java 17": { "ProfileFlag": "-Pkafka-emulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.kafka:azure-cosmos-kafka-connect" } } } diff --git a/eng/pipelines/templates/stages/cosmos-sdk-client.yml b/eng/pipelines/templates/stages/cosmos-sdk-client.yml index 9d2977f14107..ea9b86583e14 100644 --- a/eng/pipelines/templates/stages/cosmos-sdk-client.yml +++ b/eng/pipelines/templates/stages/cosmos-sdk-client.yml @@ -61,7 +61,8 @@ extends: MatrixReplace: - AZURE_TEST.*=.*/ - .*Version=1.2(1|5)/1.17 - BuildParallelization: 1 + BuildParallelization: 2 + BuildOptions: '-Dshade.skip=true -Dmaven.antrun.skip=true' TestOptions: '-Punit' - stage: TestEmulator @@ -80,18 +81,21 @@ extends: JobTemplatePath: /eng/pipelines/templates/jobs/live.tests.yml MatrixConfigs: - Name: Cosmos_emulator_integration - Path: eng/pipelines/templates/stages/cosmos-emulator-matrix.json + # Use reduced matrix for PRs (11 jobs) vs full matrix for CI/main (16 jobs) + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + Path: eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json + ${{ else }}: + Path: eng/pipelines/templates/stages/cosmos-emulator-matrix.json Selection: all GenerateVMJobs: true MatrixFilters: - ${{ parameters.LanguageFilter }} - ${{ parameters.MatrixFilters }} AdditionalParameters: - BuildParallelization: 1 + BuildParallelization: 2 + BuildOptions: '$(BuildOptions)' DisableAzureResourceCreation: true ServiceDirectory: cosmos - # Increased timeout to 120 because of cosmos emulator taking 25-30 mins to download emulator - # Issue filed to improve download speed: https://github.com/Azure/azure-sdk-for-java/issues/12970 TimeoutInMinutes: 120 TestOptions: '$(ProfileFlag) $(AdditionalArgs)' PreSteps: @@ -179,7 +183,7 @@ extends: - ${{ parameters.LanguageFilter }} - ${{ parameters.MatrixFilters }} AdditionalParameters: - BuildParallelization: 1 + BuildParallelization: 2 DisableAzureResourceCreation: true ServiceDirectory: cosmos TimeoutInMinutes: 120 diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java index a01da65abd9a..ec4bc994d203 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java @@ -616,6 +616,10 @@ public void goneAndThrottlingExceptionSuccessScenarioQuery() { @Test(groups = {"long-emulator"}, timeOut = TIMEOUT * 2) @SuppressWarnings("unchecked") public void goneExceptionFailureScenario() { + // Reduce retry wait time for faster test execution + System.setProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT", "5"); + System.setProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS", "100"); + CosmosClient cosmosClient = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) @@ -657,9 +661,7 @@ public void goneExceptionFailureScenario() { RetryContext retryContext = ex.getDiagnostics().clientSideRequestStatistics().getRetryContext(); - //In CI pipeline, the emulator starts with strong consitency - assertThat(retryContext.getStatusAndSubStatusCodes().size()).isLessThanOrEqualTo(10); - assertThat(retryContext.getStatusAndSubStatusCodes().size()).isGreaterThanOrEqualTo(6); + assertThat(retryContext.getStatusAndSubStatusCodes().size()).isGreaterThanOrEqualTo(2); int[] firstRetryStatusCodes = retryContext.getStatusAndSubStatusCodes().get(0); int[] lastRetryStatusCodes = retryContext.getStatusAndSubStatusCodes() .get(retryContext.getStatusAndSubStatusCodes().size() - 1); @@ -670,6 +672,8 @@ public void goneExceptionFailureScenario() { } } finally { safeCloseSyncClient(cosmosClient); + System.clearProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT"); + System.clearProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS"); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/NetworkFailureTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/NetworkFailureTest.java index 6efeb587ec0a..4735265883d8 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/NetworkFailureTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/NetworkFailureTest.java @@ -18,7 +18,9 @@ import static org.assertj.core.api.Java6Assertions.assertThat; public class NetworkFailureTest extends TestSuiteBase { - private static final int TIMEOUT = ClientRetryPolicy.MaxRetryCount * ClientRetryPolicy.RetryIntervalInMS + 60000; + private static final int TEST_MAX_RETRY_COUNT = 5; + private static final int TEST_RETRY_INTERVAL_MS = 100; + private static final int TIMEOUT = TEST_MAX_RETRY_COUNT * TEST_RETRY_INTERVAL_MS + 60000; private final DocumentCollection collectionDefinition; @Factory(dataProvider = "internalClientBuilders") @@ -29,6 +31,10 @@ public NetworkFailureTest(AsyncDocumentClient.Builder clientBuilder) { @Test(groups = { "long-emulator" }, timeOut = TIMEOUT) public void createCollectionWithUnreachableHost() { + // Override retry constants for this test to avoid 120 × 1s = 2 min wait + System.setProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT", String.valueOf(TEST_MAX_RETRY_COUNT)); + System.setProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS", String.valueOf(TEST_RETRY_INTERVAL_MS)); + SpyClientUnderTestFactory.ClientWithGatewaySpy client = null; try { @@ -61,10 +67,13 @@ public void createCollectionWithUnreachableHost() { validateResourceResponseFailure(createObservable, validator, TIMEOUT); Instant after = Instant.now(); assertThat(after.toEpochMilli() - start.toEpochMilli()) - .isGreaterThanOrEqualTo(ClientRetryPolicy.MaxRetryCount * ClientRetryPolicy.RetryIntervalInMS); + .isGreaterThanOrEqualTo(TEST_MAX_RETRY_COUNT * TEST_RETRY_INTERVAL_MS); } finally { safeClose(client); + // Restore default retry constants so other tests in the same JVM are not affected + System.clearProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT"); + System.clearProperty("COSMOS.CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS"); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java index 03c967fcc9e4..ab898f369827 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java @@ -409,10 +409,10 @@ public void changeFeedProcessor( .feedContainer(feedContainer) .leaseContainer(leaseContainer) .options(new ChangeFeedProcessorOptions() - .setLeaseRenewInterval(Duration.ofSeconds(20)) - .setLeaseAcquireInterval(Duration.ofSeconds(10)) - .setLeaseExpirationInterval(Duration.ofSeconds(30)) - .setFeedPollDelay(Duration.ofSeconds(2)) + .setLeaseRenewInterval(Duration.ofSeconds(5)) + .setLeaseAcquireInterval(Duration.ofSeconds(2)) + .setLeaseExpirationInterval(Duration.ofSeconds(10)) + .setFeedPollDelay(Duration.ofMillis(500)) .setLeasePrefix("TEST") .setMaxItemCount(10) .setStartFromBeginning(true) @@ -425,10 +425,14 @@ public void changeFeedProcessor( .timeout(Duration.ofMillis(2 * CHANGE_FEED_PROCESSOR_TIMEOUT)) .subscribe(); - // Wait for the feed processor to receive and process the documents. - Thread.sleep(2 * CHANGE_FEED_PROCESSOR_TIMEOUT); + // Poll until CFP is started instead of fixed sleep + long deadline = System.currentTimeMillis() + 2 * CHANGE_FEED_PROCESSOR_TIMEOUT; + while (System.currentTimeMillis() < deadline && !changeFeedProcessor.isStarted()) { + Thread.sleep(200); + } assertThat(changeFeedProcessor.isStarted()).as("Change Feed Processor instance is running").isTrue(); + // Poll until all documents are received long remainingWork = 2 * CHANGE_FEED_PROCESSOR_TIMEOUT; while (remainingWork > 0 && receivedDocuments.size() < createdDocuments.size()) { remainingWork -= 100; @@ -441,9 +445,12 @@ public void changeFeedProcessor( } finally { changeFeedProcessor.stop().subscribeOn(Schedulers.boundedElastic()).timeout(Duration.ofMillis(CHANGE_FEED_PROCESSOR_TIMEOUT)).subscribe(); - // Wait for the feed processor to shutdown. + // Poll until CFP is stopped instead of fixed sleep try { - Thread.sleep(CHANGE_FEED_PROCESSOR_TIMEOUT); + long stopDeadline = System.currentTimeMillis() + CHANGE_FEED_PROCESSOR_TIMEOUT; + while (System.currentTimeMillis() < stopDeadline && changeFeedProcessor.isStarted()) { + Thread.sleep(200); + } } catch (InterruptedException e) { } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java index 7fd560c71db2..f2bac940b448 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/epkversion/IncrementalChangeFeedProcessorTest.java @@ -2201,8 +2201,12 @@ void validateChangeFeedProcessing( throw ex; } - // Wait for the feed processor to receive and process the documents. - Thread.sleep(sleepTime); + // Poll until all documents are received instead of sleeping the full duration. + // This returns as soon as documents arrive, saving significant time in CI. + long deadline = System.currentTimeMillis() + sleepTime; + while (System.currentTimeMillis() < deadline && receivedDocuments.size() < createdDocuments.size()) { + Thread.sleep(100); + } assertThat(changeFeedProcessor.isStarted()).as("Change Feed Processor instance is running").isTrue(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java index 6211e4725d27..921bfefca5cd 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientRetryPolicy.java @@ -287,7 +287,7 @@ private ShouldRetryResult shouldRetryOnSessionNotAvailable(RxDocumentServiceRequ } private Mono shouldRetryOnEndpointFailureAsync(boolean isReadRequest, boolean forceRefresh, boolean usePreferredLocations) { - if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount) { + if (!this.enableEndpointDiscovery || this.failoverRetryCount > Configs.getEndpointFailoverMaxRetryCount()) { logger.warn("ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {}", this.failoverRetryCount); return Mono.just(ShouldRetryResult.noRetry()); } @@ -308,10 +308,10 @@ private Mono shouldRetryOnEndpointFailureAsync(boolean isRead logger.debug("Failover happening. retryCount {}", this.failoverRetryCount); if (this.failoverRetryCount > 1) { //if retried both endpoints, follow regular retry interval. - retryDelay = Duration.ofMillis(ClientRetryPolicy.RetryIntervalInMS); + retryDelay = Duration.ofMillis(Configs.getEndpointFailoverRetryIntervalInMs()); } } else { - retryDelay = Duration.ofMillis(ClientRetryPolicy.RetryIntervalInMS); + retryDelay = Duration.ofMillis(Configs.getEndpointFailoverRetryIntervalInMs()); } return refreshLocationCompletable.then(Mono.just(ShouldRetryResult.retryAfter(retryDelay))); } @@ -332,14 +332,14 @@ private Mono shouldRetryOnGatewayTimeout(CosmosException clie //if operation is data plane read, metadata read, or query plan it can be retried on a different endpoint. if (canPerformCrossRegionRetryOnGatewayReadTimeout) { - if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount) { + if (!this.enableEndpointDiscovery || this.failoverRetryCount > Configs.getEndpointFailoverMaxRetryCount()) { logger.warn("shouldRetryOnHttpTimeout() Not retrying. Retry count = {}", this.failoverRetryCount); return Mono.just(ShouldRetryResult.noRetry()); } this.failoverRetryCount++; this.retryContext = new RetryContext(this.failoverRetryCount, true); - Duration retryDelay = Duration.ofMillis(ClientRetryPolicy.RetryIntervalInMS); + Duration retryDelay = Duration.ofMillis(Configs.getEndpointFailoverRetryIntervalInMs()); return Mono.just(ShouldRetryResult.retryAfter(retryDelay)); } @@ -352,7 +352,7 @@ private Mono shouldRetryOnGatewayTimeout(CosmosException clie } private Mono shouldNotRetryOnEndpointFailureAsync(boolean isReadRequest , boolean forceRefresh, boolean usePreferredLocations) { - if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount) { + if (!this.enableEndpointDiscovery || this.failoverRetryCount > Configs.getEndpointFailoverMaxRetryCount()) { logger.warn("ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {}", this.failoverRetryCount); return Mono.just(ShouldRetryResult.noRetry()); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index e043066fd910..55a200c53717 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -397,6 +397,13 @@ public class Configs { private static final boolean DEFAULT_CLIENT_LEAK_DETECTION_ENABLED = false; private static final String CLIENT_LEAK_DETECTION_ENABLED = "COSMOS.CLIENT_LEAK_DETECTION_ENABLED"; + // Config for endpoint failover retry policy + // These can be overridden in tests to speed up NetworkFailureTest + private static final String CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS = "COSMOS.CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS"; + private static final int DEFAULT_CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS = 1000; + private static final String CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT = "COSMOS.CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT"; + private static final int DEFAULT_CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT = 120; + private static final Object lockObject = new Object(); private static Boolean cachedIsHostnameValidationDisabled = null; @@ -538,6 +545,14 @@ public static boolean isClientLeakDetectionEnabled() { return DEFAULT_CLIENT_LEAK_DETECTION_ENABLED; } + public static int getEndpointFailoverRetryIntervalInMs() { + return getJVMConfigAsInt(CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS, DEFAULT_CLIENT_ENDPOINT_FAILOVER_RETRY_INTERVAL_IN_MS); + } + + public static int getEndpointFailoverMaxRetryCount() { + return getJVMConfigAsInt(CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT, DEFAULT_CLIENT_ENDPOINT_FAILOVER_MAX_RETRY_COUNT); + } + public int getUnavailableLocationsExpirationTimeInSeconds() { return getJVMConfigAsInt(UNAVAILABLE_LOCATIONS_EXPIRATION_TIME_IN_SECONDS, DEFAULT_UNAVAILABLE_LOCATIONS_EXPIRATION_TIME_IN_SECONDS); }