Kushagrathapar/optimize cosmos ci shared build

eng/pipelines/scripts/generate-project-list.ps1

@@ -4,6 +4,22 @@ $additionalModulesList = @()
 
 . "${PSScriptRoot}/../../common/scripts/common.ps1"
 
+# If ProjectListOverride is set (e.g., from matrix variables), use it directly
+# to avoid building unnecessary modules in jobs that only test a subset.
+if ($env:PROJECTLISTOVERRIDE -and $env:PROJECTLISTOVERRIDE -notlike '*ProjectListOverride*') {
+  $projects = $env:PROJECTLISTOVERRIDE
+  Write-Host "Using ProjectListOverride = $projects"
+  Write-Host "##vso[task.setvariable variable=ProjectList;]$projects"
+  Write-Host "##vso[task.setvariable variable=ArtifactsList;]$projects"
+  Write-Host "##vso[task.setvariable variable=AdditionalModulesList;]"
+
+  $sha256 = new-object -TypeName System.Security.Cryptography.SHA256Managed
+  $utf8 = new-object -TypeName System.Text.UTF8Encoding
+  $projectListSha256 = [Convert]::ToBase64String($sha256.ComputeHash($utf8.GetBytes($projects)))
+  Write-Host "##vso[task.setvariable variable=ProjectListSha256;]$projectListSha256"
+  return
+}
+
 if ($env:ARTIFACTSJSON -and $env:ARTIFACTSJSON -notlike '*ArtifactsJson*') {
   $artifacts = $env:ARTIFACTSJSON | ConvertFrom-Json
   foreach ($artifact in $artifacts) {

eng/pipelines/templates/jobs/ci.tests.yml

@@ -40,6 +40,9 @@ parameters:
   - name: BuildParallelization
     type: string
     default: '2C'
+  - name: BuildOptions
+    type: string
+    default: ''
   - name: AdditionalLintingOptions
     type: string
     default: ''
@@ -145,6 +148,7 @@ jobs:
           TestOptions: ${{ parameters.TestOptions }}
           TestParallelization: ${{ parameters.TestParallelization }}
           BuildParallelization: ${{ parameters.BuildParallelization }}
+          BuildOptions: ${{ parameters.BuildOptions }}
           TestEnvVars: ${{ parameters.EnvVars }}
           VersionOverride: ${{ parameters.VersionOverride }}
 

eng/pipelines/templates/jobs/ci.yml

@@ -40,6 +40,9 @@ parameters:
   - name: BuildParallelization
     type: string
     default: '2C'
+  - name: BuildOptions
+    type: string
+    default: ''
   - name: TestGoals
     type: string
     default: $(TestGoals)
@@ -503,6 +506,7 @@ jobs:
         TestOptions: ${{ parameters.TestOptions }}
         TestParallelization: ${{ parameters.TestParallelization }}
         BuildParallelization: ${{ parameters.BuildParallelization }}
+        BuildOptions: ${{ parameters.BuildOptions }}
         AdditionalLintingOptions: ${{ parameters.AdditionalLintingOptions }}
         TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
         EnvVars: ${{ parameters.EnvVars }}

eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json

@@ -0,0 +1,110 @@
+{
+  "displayNames": {
+    "clean verify": ""
+  },
+  "matrix": {
+    "Agent": {
+      "windows-2022": {
+        "OSVmImage": "env:WINDOWSVMIMAGE",
+        "Pool": "env:WINDOWSPOOL"
+      }
+    },
+    "TestGoals": "clean verify",
+    "EmulatorConfig": {
+      "Emulator Only Integration Tests Tcp - Java 8": {
+        "ProfileFlag": "-Pemulator",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.8",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true",
+        "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true"
+      },
+      "Emulator Only Integration Tests Tcp - Java 17": {
+        "ProfileFlag": "-Pemulator",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Strong\"]",
+        "JavaTestVersion": "1.17",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true",
+        "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true"
+      },
+      "Long Emulator Only Integration Tests Tcp - Java 8": {
+        "ProfileFlag": "-Plong-emulator",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.8",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true",
+        "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true"
+      },
+      "Long Emulator Only Integration Tests Tcp - Java 17": {
+        "ProfileFlag": "-Plong-emulator",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Strong\"]",
+        "JavaTestVersion": "1.17",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true",
+        "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true"
+      },
+      "Encryption Emulator Only Integration Tests": {
+        "ProfileFlag": "-Pencryption-integration",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.8",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true",
+        "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true"
+      },
+      "Spark 3.3 Integration Tests targeting Cosmos Emulator - Java 8'": {
+        "ProfileFlag": "-Dspark-e2e_3-3",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.8",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true",
+        "BuildOptions": "",
+        "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12"
+      },
+      "Spark 3.4 Integration Tests targeting Cosmos Emulator - Java 11'": {
+        "ProfileFlag": "-Dspark-e2e_3-4",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.11",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true",
+        "BuildOptions": "",
+        "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-4_2-12"
+      },
+      "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 17'": {
+        "ProfileFlag": "-Dspark-e2e_3-5_2-12",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.17",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true",
+        "BuildOptions": "",
+        "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-12"
+      },
+      "Spark 3.5, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": {
+        "ProfileFlag": "-Dspark-e2e_3-5_2-13",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.17",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true",
+        "BuildOptions": "",
+        "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-13"
+      },
+      "Spark 4.0, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 21'": {
+        "ProfileFlag": "-Dspark-e2e_4-0_2-13",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.21",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true",
+        "BuildOptions": "",
+        "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13"
+      },
+      "Kafka Integration Tests targeting Cosmos Emulator - Java 17": {
+        "ProfileFlag": "-Pkafka-emulator",
+        "PROTOCOLS": "[\"Tcp\"]",
+        "DESIRED_CONSISTENCIES": "[\"Session\"]",
+        "JavaTestVersion": "1.17",
+        "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true",
+        "BuildOptions": "",
+        "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.kafka:azure-cosmos-kafka-connect"
+      }
+    }
+  }
+}