diff --git a/modules_combined/kafka_load_and_array_parse/Dockerfile b/modules_combined/kafka_load_and_array_parse/Dockerfile index 896572273..669bf187e 100644 --- a/modules_combined/kafka_load_and_array_parse/Dockerfile +++ b/modules_combined/kafka_load_and_array_parse/Dockerfile @@ -5,7 +5,7 @@ # docker build -t neon-is-kfka-load-arry-pars:latest -f ./modules_combined/kafka_load_and_array_parse/Dockerfile . # ### -FROM us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 +FROM us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 ARG MODULE_DIR="modules" ARG APP_DIR="array_parser" diff --git a/pipe/aepg600m/aepg600m_data_source_kafka.yaml b/pipe/aepg600m/aepg600m_data_source_kafka.yaml index 5514dd0d1..4471e7b4c 100644 --- a/pipe/aepg600m/aepg600m_data_source_kafka.yaml +++ b/pipe/aepg600m/aepg600m_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: aepg600m_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/cmp22/cmp22_data_source_kafka.yaml b/pipe/cmp22/cmp22_data_source_kafka.yaml index 049aa3ea5..f1659c918 100644 --- a/pipe/cmp22/cmp22_data_source_kafka.yaml +++ b/pipe/cmp22/cmp22_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: cmp22_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/csat3/csat3_data_source_kafka.yaml b/pipe/csat3/csat3_data_source_kafka.yaml index b4ad34ea0..a8c74561d 100644 --- a/pipe/csat3/csat3_data_source_kafka.yaml +++ b/pipe/csat3/csat3_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: csat3_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/csat3b/csat3b_data_source_kafka.yaml b/pipe/csat3b/csat3b_data_source_kafka.yaml index 33c26c9f0..4bd99c132 100644 --- a/pipe/csat3b/csat3b_data_source_kafka.yaml +++ b/pipe/csat3b/csat3b_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: csat3b_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/drx8533ep/drx8533ep_data_parser.yaml b/pipe/drx8533ep/drx8533ep_data_parser.yaml index 4400ad6c2..e9cefe008 100644 --- a/pipe/drx8533ep/drx8533ep_data_parser.yaml +++ b/pipe/drx8533ep/drx8533ep_data_parser.yaml @@ -1,7 +1,7 @@ pipeline: name: drx8533ep_data_parser transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 env: # if use default PARSED_START_INDEX and PARSED_END_INDEX, parse all elements in parse_field # if use default for FIELD_START_INDEX and FIELD_END_INDEX, diff --git a/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml b/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml index dcfceaa81..19b59028b 100644 --- a/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml +++ b/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: drx8533ep_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/dualfan/dualfan_data_source_kafka.yaml b/pipe/dualfan/dualfan_data_source_kafka.yaml index 6fe83906c..379a29b7c 100644 --- a/pipe/dualfan/dualfan_data_source_kafka.yaml +++ b/pipe/dualfan/dualfan_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: dualfan_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/ecse/ecse_event_data_source_kafka.yaml b/pipe/ecse/ecse_event_data_source_kafka.yaml index 14deb213f..95c5128f1 100644 --- a/pipe/ecse/ecse_event_data_source_kafka.yaml +++ b/pipe/ecse/ecse_event_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ecse_event_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -29,6 +29,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/enviroscan/enviroscan_data_source_kafka.yaml b/pipe/enviroscan/enviroscan_data_source_kafka.yaml index d51015713..34add973c 100644 --- a/pipe/enviroscan/enviroscan_data_source_kafka.yaml +++ b/pipe/enviroscan/enviroscan_data_source_kafka.yaml @@ -34,6 +34,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/exo2/exo2_data_source_kafka.yaml b/pipe/exo2/exo2_data_source_kafka.yaml index 193151f9b..15e9f12f9 100644 --- a/pipe/exo2/exo2_data_source_kafka.yaml +++ b/pipe/exo2/exo2_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -32,6 +32,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/exo2/exo2_hourly_data_source_kafka.yaml b/pipe/exo2/exo2_hourly_data_source_kafka.yaml index 86c233cec..782f36eaf 100644 --- a/pipe/exo2/exo2_hourly_data_source_kafka.yaml +++ b/pipe/exo2/exo2_hourly_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_hourly_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -32,6 +32,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/exo2/exo2_step_data_source_kafka.yaml b/pipe/exo2/exo2_step_data_source_kafka.yaml index af9df3bd1..e6223be99 100644 --- a/pipe/exo2/exo2_step_data_source_kafka.yaml +++ b/pipe/exo2/exo2_step_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_step_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -32,6 +32,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/g2131i/g2131i_data_source_kafka.yaml b/pipe/g2131i/g2131i_data_source_kafka.yaml index 56b67acb6..9988b55b7 100644 --- a/pipe/g2131i/g2131i_data_source_kafka.yaml +++ b/pipe/g2131i/g2131i_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: g2131i_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/gmp343/gmp343_data_source_kafka.yaml b/pipe/gmp343/gmp343_data_source_kafka.yaml index cc72a8f88..defa9f941 100644 --- a/pipe/gmp343/gmp343_data_source_kafka.yaml +++ b/pipe/gmp343/gmp343_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: gmp343_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml b/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml index 25a6e9e07..6c04e5aba 100644 --- a/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml +++ b/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hfp01sc_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/hmp155/hmp155_data_source_kafka.yaml b/pipe/hmp155/hmp155_data_source_kafka.yaml index 6bea24e20..f8a990013 100644 --- a/pipe/hmp155/hmp155_data_source_kafka.yaml +++ b/pipe/hmp155/hmp155_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hmp155_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/hmr3300/hmr3300_data_source_kafka.yaml b/pipe/hmr3300/hmr3300_data_source_kafka.yaml index d64cd2bc9..0a04dabb5 100644 --- a/pipe/hmr3300/hmr3300_data_source_kafka.yaml +++ b/pipe/hmr3300/hmr3300_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hmr3300_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/l2130i/l2130i_data_source_kafka.yaml b/pipe/l2130i/l2130i_data_source_kafka.yaml index f4abb2031..c972ac4b0 100644 --- a/pipe/l2130i/l2130i_data_source_kafka.yaml +++ b/pipe/l2130i/l2130i_data_source_kafka.yaml @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li191r/li191r_data_source_kafka.yaml b/pipe/li191r/li191r_data_source_kafka.yaml index 6ee6558c2..729dff75f 100644 --- a/pipe/li191r/li191r_data_source_kafka.yaml +++ b/pipe/li191r/li191r_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li191r_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li192sa/li192sa_data_source_kafka.yaml b/pipe/li192sa/li192sa_data_source_kafka.yaml index 19c13a1bf..a4d86058d 100644 --- a/pipe/li192sa/li192sa_data_source_kafka.yaml +++ b/pipe/li192sa/li192sa_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li192sa_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li7200/li7200_data_source_kafka.yaml b/pipe/li7200/li7200_data_source_kafka.yaml index 3da5dad56..2821b7a3b 100644 --- a/pipe/li7200/li7200_data_source_kafka.yaml +++ b/pipe/li7200/li7200_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li7200_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li840a/li840a_data_source_kafka.yaml b/pipe/li840a/li840a_data_source_kafka.yaml index a8f924c7e..6fa6c5209 100644 --- a/pipe/li840a/li840a_data_source_kafka.yaml +++ b/pipe/li840a/li840a_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li840a_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li850/li850_data_source_kafka.yaml b/pipe/li850/li850_data_source_kafka.yaml index 84aaa28e9..4dbf7153a 100644 --- a/pipe/li850/li850_data_source_kafka.yaml +++ b/pipe/li850/li850_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li850_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/mcseries/mcseries_data_source_kafka.yaml b/pipe/mcseries/mcseries_data_source_kafka.yaml index 47b250146..13eb3cf8a 100644 --- a/pipe/mcseries/mcseries_data_source_kafka.yaml +++ b/pipe/mcseries/mcseries_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mcseries_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/metone370380/metone370380_data_source_kafka.yaml b/pipe/metone370380/metone370380_data_source_kafka.yaml index 21c33450f..a5b9cad9a 100644 --- a/pipe/metone370380/metone370380_data_source_kafka.yaml +++ b/pipe/metone370380/metone370380_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: metone370380_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml b/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml index 4c260f82e..15d124d83 100644 --- a/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml +++ b/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mti300ahrs_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/mwseries/mwseries_data_source_kafka.yaml b/pipe/mwseries/mwseries_data_source_kafka.yaml index 4643e0552..3c4f10dea 100644 --- a/pipe/mwseries/mwseries_data_source_kafka.yaml +++ b/pipe/mwseries/mwseries_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mwseries_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml b/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml index a2188b473..3023b72f7 100644 --- a/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml +++ b/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ncon127tm_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/nr01/nr01_data_source_kafka.yaml b/pipe/nr01/nr01_data_source_kafka.yaml index f4de6c156..ba8a70d79 100644 --- a/pipe/nr01/nr01_data_source_kafka.yaml +++ b/pipe/nr01/nr01_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: nr01_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/picarro3way/picarro3way_data_source_kafka.yaml b/pipe/picarro3way/picarro3way_data_source_kafka.yaml index cc4a61fbf..d0053dc47 100644 --- a/pipe/picarro3way/picarro3way_data_source_kafka.yaml +++ b/pipe/picarro3way/picarro3way_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: picarro3way_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pluvio/pluvio_data_source_kafka.yaml b/pipe/pluvio/pluvio_data_source_kafka.yaml index a8f927f27..f6bd4c0c3 100644 --- a/pipe/pluvio/pluvio_data_source_kafka.yaml +++ b/pipe/pluvio/pluvio_data_source_kafka.yaml @@ -37,6 +37,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pqs1/pqs1_data_source_kafka.yaml b/pipe/pqs1/pqs1_data_source_kafka.yaml index c33aa8a48..24f7dd8a7 100644 --- a/pipe/pqs1/pqs1_data_source_kafka.yaml +++ b/pipe/pqs1/pqs1_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pqs1_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml b/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml index 664c3bf10..242bf3ad1 100644 --- a/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml +++ b/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pressuretransducer_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/prt/prt_data_source_kafka.yaml b/pipe/prt/prt_data_source_kafka.yaml index 7b575ec30..afc200430 100644 --- a/pipe/prt/prt_data_source_kafka.yaml +++ b/pipe/prt/prt_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: prt_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/prtncon/prtncon_data_source_kafka.yaml b/pipe/prtncon/prtncon_data_source_kafka.yaml index 98be52203..19ff6320c 100644 --- a/pipe/prtncon/prtncon_data_source_kafka.yaml +++ b/pipe/prtncon/prtncon_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: prtncon_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/ptb330a/ptb330a_data_source_kafka.yaml b/pipe/ptb330a/ptb330a_data_source_kafka.yaml index 513abca1d..6d9141e3a 100644 --- a/pipe/ptb330a/ptb330a_data_source_kafka.yaml +++ b/pipe/ptb330a/ptb330a_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ptb330a_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pump/pump_data_source_kafka.yaml b/pipe/pump/pump_data_source_kafka.yaml index 585e0e0c0..99f070110 100644 --- a/pipe/pump/pump_data_source_kafka.yaml +++ b/pipe/pump/pump_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pump_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/rmyoung/rmyoung_data_source_kafka.yaml b/pipe/rmyoung/rmyoung_data_source_kafka.yaml index 62db5d5a7..d1261c8b1 100644 --- a/pipe/rmyoung/rmyoung_data_source_kafka.yaml +++ b/pipe/rmyoung/rmyoung_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: rmyoung_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml b/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml index b72d6e40b..187a1593a 100644 --- a/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml +++ b/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml @@ -2,7 +2,7 @@ pipeline: name: ONEOFF_level0_bucket_compact transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 cmd: - sh - "-c" diff --git a/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml b/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml index 69c213087..c12f6990a 100644 --- a/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml +++ b/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml @@ -2,7 +2,7 @@ pipeline: name: li7200_parse_kafka_raw_data transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 env: # if use default PARSED_START_INDEX and PARSED_END_INDEX, parse all elements in parse_field # if use default for FIELD_START_INDEX and FIELD_END_INDEX, diff --git a/pipe/si111/si111_data_source_kafka.yaml b/pipe/si111/si111_data_source_kafka.yaml index 26ad86b5b..eba3e322b 100644 --- a/pipe/si111/si111_data_source_kafka.yaml +++ b/pipe/si111/si111_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: si111_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/solenoid/solenoid_data_source_kafka.yaml b/pipe/solenoid/solenoid_data_source_kafka.yaml index 61bc3582f..3f1afa91d 100644 --- a/pipe/solenoid/solenoid_data_source_kafka.yaml +++ b/pipe/solenoid/solenoid_data_source_kafka.yaml @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/spn1/spn1_data_source_kafka.yaml b/pipe/spn1/spn1_data_source_kafka.yaml index ee629b243..b4040c725 100644 --- a/pipe/spn1/spn1_data_source_kafka.yaml +++ b/pipe/spn1/spn1_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: spn1_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/sq500ss/sq500ss_data_source_kafka.yaml b/pipe/sq500ss/sq500ss_data_source_kafka.yaml index 1b67e77b1..cff939d5a 100644 --- a/pipe/sq500ss/sq500ss_data_source_kafka.yaml +++ b/pipe/sq500ss/sq500ss_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: sq500ss_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/stevenshp/stevenshp_data_source_kafka.yaml b/pipe/stevenshp/stevenshp_data_source_kafka.yaml index 2232e3fae..1e952c1f8 100644 --- a/pipe/stevenshp/stevenshp_data_source_kafka.yaml +++ b/pipe/stevenshp/stevenshp_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: stevenshp_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/sunav2/sunav2_data_source_kafka.yaml b/pipe/sunav2/sunav2_data_source_kafka.yaml index 1e1886c43..a563440e7 100644 --- a/pipe/sunav2/sunav2_data_source_kafka.yaml +++ b/pipe/sunav2/sunav2_data_source_kafka.yaml @@ -34,6 +34,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/t7610/t7610_data_source_kafka.yaml b/pipe/t7610/t7610_data_source_kafka.yaml index 1597816d6..a63dad128 100644 --- a/pipe/t7610/t7610_data_source_kafka.yaml +++ b/pipe/t7610/t7610_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: t7610_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/tchain/tchain_data_source_kafka.yaml b/pipe/tchain/tchain_data_source_kafka.yaml index d49c6d10f..a8104cca0 100644 --- a/pipe/tchain/tchain_data_source_kafka.yaml +++ b/pipe/tchain/tchain_data_source_kafka.yaml @@ -34,6 +34,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/troll/troll_data_source_kafka.yaml b/pipe/troll/troll_data_source_kafka.yaml index a7f692ccd..8a81a611c 100644 --- a/pipe/troll/troll_data_source_kafka.yaml +++ b/pipe/troll/troll_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: troll_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/windobserverii/windobserverii_data_source_kafka.yaml b/pipe/windobserverii/windobserverii_data_source_kafka.yaml index 1404986ec..f89545395 100644 --- a/pipe/windobserverii/windobserverii_data_source_kafka.yaml +++ b/pipe/windobserverii/windobserverii_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: windobserverii_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml b/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml index 7485da034..ddc9c00f7 100644 --- a/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml +++ b/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: windobserverii_extreme_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET