From 5b9849af9fe694f8dc55e74c3d9e8938a08f7a77 Mon Sep 17 00:00:00 2001 From: Steve Jacobs Date: Mon, 2 Mar 2026 12:04:49 -0500 Subject: [PATCH 1/2] Update pipelines to use new compact log --- modules_combined/kafka_load_and_array_parse/Dockerfile | 2 +- pipe/aepg600m/aepg600m_data_source_kafka.yaml | 5 ++++- pipe/cmp22/cmp22_data_source_kafka.yaml | 5 ++++- pipe/csat3/csat3_data_source_kafka.yaml | 5 ++++- pipe/csat3b/csat3b_data_source_kafka.yaml | 5 ++++- pipe/drx8533ep/drx8533ep_data_parser.yaml | 2 +- pipe/drx8533ep/drx8533ep_data_source_kafka.yaml | 5 ++++- pipe/dualfan/dualfan_data_source_kafka.yaml | 5 ++++- pipe/ecse/ecse_event_data_source_kafka.yaml | 5 ++++- pipe/enviroscan/enviroscan_data_source_kafka.yaml | 3 +++ pipe/exo2/exo2_data_source_kafka.yaml | 5 ++++- pipe/exo2/exo2_hourly_data_source_kafka.yaml | 5 ++++- pipe/exo2/exo2_step_data_source_kafka.yaml | 5 ++++- pipe/g2131i/g2131i_data_source_kafka.yaml | 5 ++++- pipe/gmp343/gmp343_data_source_kafka.yaml | 5 ++++- pipe/hfp01sc/hfp01sc_data_source_kafka.yaml | 5 ++++- pipe/hmp155/hmp155_data_source_kafka.yaml | 5 ++++- pipe/hmr3300/hmr3300_data_source_kafka.yaml | 5 ++++- pipe/l2130i/l2130i_data_source_kafka.yaml | 3 +++ pipe/li191r/li191r_data_source_kafka.yaml | 5 ++++- pipe/li192sa/li192sa_data_source_kafka.yaml | 5 ++++- pipe/li7200/li7200_data_source_kafka.yaml | 5 ++++- pipe/li840a/li840a_data_source_kafka.yaml | 5 ++++- pipe/li850/li850_data_source_kafka.yaml | 5 ++++- pipe/mcseries/mcseries_data_source_kafka.yaml | 5 ++++- pipe/metone370380/metone370380_data_source_kafka.yaml | 5 ++++- pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml | 5 ++++- pipe/mwseries/mwseries_data_source_kafka.yaml | 5 ++++- pipe/ncon127tm/ncon127tm_data_source_kafka.yaml | 5 ++++- pipe/nr01/nr01_data_source_kafka.yaml | 5 ++++- pipe/picarro3way/picarro3way_data_source_kafka.yaml | 5 ++++- pipe/pluvio/pluvio_data_source_kafka.yaml | 3 +++ pipe/pqs1/pqs1_data_source_kafka.yaml | 5 ++++- .../pressuretransducer_data_source_kafka.yaml | 5 ++++- pipe/prt/prt_data_source_kafka.yaml | 5 ++++- pipe/prtncon/prtncon_data_source_kafka.yaml | 5 ++++- pipe/ptb330a/ptb330a_data_source_kafka.yaml | 5 ++++- pipe/pump/pump_data_source_kafka.yaml | 5 ++++- pipe/rmyoung/rmyoung_data_source_kafka.yaml | 5 ++++- pipe/sandbox/ONEOFF_level0_bucket_compact.yaml | 2 +- pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml | 2 +- pipe/si111/si111_data_source_kafka.yaml | 5 ++++- pipe/solenoid/solenoid_data_source_kafka.yaml | 3 +++ pipe/spn1/spn1_data_source_kafka.yaml | 5 ++++- pipe/sq500ss/sq500ss_data_source_kafka.yaml | 5 ++++- pipe/stevenshp/stevenshp_data_source_kafka.yaml | 5 ++++- pipe/sunav2/sunav2_data_source_kafka.yaml | 3 +++ pipe/t7610/t7610_data_source_kafka.yaml | 5 ++++- pipe/tchain/tchain_data_source_kafka.yaml | 3 +++ pipe/troll/troll_data_source_kafka.yaml | 5 ++++- pipe/windobserverii/windobserverii_data_source_kafka.yaml | 5 ++++- .../windobserverii_extreme_data_source_kafka.yaml | 5 ++++- 52 files changed, 190 insertions(+), 46 deletions(-) diff --git a/modules_combined/kafka_load_and_array_parse/Dockerfile b/modules_combined/kafka_load_and_array_parse/Dockerfile index 896572273..a0821ecf2 100644 --- a/modules_combined/kafka_load_and_array_parse/Dockerfile +++ b/modules_combined/kafka_load_and_array_parse/Dockerfile @@ -5,7 +5,7 @@ # docker build -t neon-is-kfka-load-arry-pars:latest -f ./modules_combined/kafka_load_and_array_parse/Dockerfile . # ### -FROM us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 +FROM us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 ARG MODULE_DIR="modules" ARG APP_DIR="array_parser" diff --git a/pipe/aepg600m/aepg600m_data_source_kafka.yaml b/pipe/aepg600m/aepg600m_data_source_kafka.yaml index 5514dd0d1..d84e00d50 100644 --- a/pipe/aepg600m/aepg600m_data_source_kafka.yaml +++ b/pipe/aepg600m/aepg600m_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: aepg600m_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/cmp22/cmp22_data_source_kafka.yaml b/pipe/cmp22/cmp22_data_source_kafka.yaml index 049aa3ea5..33525eff2 100644 --- a/pipe/cmp22/cmp22_data_source_kafka.yaml +++ b/pipe/cmp22/cmp22_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: cmp22_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/csat3/csat3_data_source_kafka.yaml b/pipe/csat3/csat3_data_source_kafka.yaml index b4ad34ea0..f215b66c2 100644 --- a/pipe/csat3/csat3_data_source_kafka.yaml +++ b/pipe/csat3/csat3_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: csat3_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/csat3b/csat3b_data_source_kafka.yaml b/pipe/csat3b/csat3b_data_source_kafka.yaml index 33c26c9f0..b2d4781d9 100644 --- a/pipe/csat3b/csat3b_data_source_kafka.yaml +++ b/pipe/csat3b/csat3b_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: csat3b_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/drx8533ep/drx8533ep_data_parser.yaml b/pipe/drx8533ep/drx8533ep_data_parser.yaml index 4400ad6c2..fd2774ce8 100644 --- a/pipe/drx8533ep/drx8533ep_data_parser.yaml +++ b/pipe/drx8533ep/drx8533ep_data_parser.yaml @@ -1,7 +1,7 @@ pipeline: name: drx8533ep_data_parser transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 env: # if use default PARSED_START_INDEX and PARSED_END_INDEX, parse all elements in parse_field # if use default for FIELD_START_INDEX and FIELD_END_INDEX, diff --git a/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml b/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml index dcfceaa81..0c6ee37fe 100644 --- a/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml +++ b/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: drx8533ep_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/dualfan/dualfan_data_source_kafka.yaml b/pipe/dualfan/dualfan_data_source_kafka.yaml index 6fe83906c..bcaf0a4d4 100644 --- a/pipe/dualfan/dualfan_data_source_kafka.yaml +++ b/pipe/dualfan/dualfan_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: dualfan_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/ecse/ecse_event_data_source_kafka.yaml b/pipe/ecse/ecse_event_data_source_kafka.yaml index 14deb213f..f7f13b9df 100644 --- a/pipe/ecse/ecse_event_data_source_kafka.yaml +++ b/pipe/ecse/ecse_event_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ecse_event_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -29,6 +29,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/enviroscan/enviroscan_data_source_kafka.yaml b/pipe/enviroscan/enviroscan_data_source_kafka.yaml index d51015713..34add973c 100644 --- a/pipe/enviroscan/enviroscan_data_source_kafka.yaml +++ b/pipe/enviroscan/enviroscan_data_source_kafka.yaml @@ -34,6 +34,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/exo2/exo2_data_source_kafka.yaml b/pipe/exo2/exo2_data_source_kafka.yaml index 193151f9b..f6338acda 100644 --- a/pipe/exo2/exo2_data_source_kafka.yaml +++ b/pipe/exo2/exo2_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -32,6 +32,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/exo2/exo2_hourly_data_source_kafka.yaml b/pipe/exo2/exo2_hourly_data_source_kafka.yaml index 86c233cec..71237f2c0 100644 --- a/pipe/exo2/exo2_hourly_data_source_kafka.yaml +++ b/pipe/exo2/exo2_hourly_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_hourly_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -32,6 +32,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/exo2/exo2_step_data_source_kafka.yaml b/pipe/exo2/exo2_step_data_source_kafka.yaml index af9df3bd1..067a5046d 100644 --- a/pipe/exo2/exo2_step_data_source_kafka.yaml +++ b/pipe/exo2/exo2_step_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_step_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -32,6 +32,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/g2131i/g2131i_data_source_kafka.yaml b/pipe/g2131i/g2131i_data_source_kafka.yaml index 56b67acb6..7252ab78b 100644 --- a/pipe/g2131i/g2131i_data_source_kafka.yaml +++ b/pipe/g2131i/g2131i_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: g2131i_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/gmp343/gmp343_data_source_kafka.yaml b/pipe/gmp343/gmp343_data_source_kafka.yaml index cc72a8f88..f8f1555cb 100644 --- a/pipe/gmp343/gmp343_data_source_kafka.yaml +++ b/pipe/gmp343/gmp343_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: gmp343_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml b/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml index 25a6e9e07..290e42297 100644 --- a/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml +++ b/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hfp01sc_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/hmp155/hmp155_data_source_kafka.yaml b/pipe/hmp155/hmp155_data_source_kafka.yaml index 6bea24e20..e33720f17 100644 --- a/pipe/hmp155/hmp155_data_source_kafka.yaml +++ b/pipe/hmp155/hmp155_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hmp155_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/hmr3300/hmr3300_data_source_kafka.yaml b/pipe/hmr3300/hmr3300_data_source_kafka.yaml index d64cd2bc9..466d28deb 100644 --- a/pipe/hmr3300/hmr3300_data_source_kafka.yaml +++ b/pipe/hmr3300/hmr3300_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hmr3300_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/l2130i/l2130i_data_source_kafka.yaml b/pipe/l2130i/l2130i_data_source_kafka.yaml index f4abb2031..c972ac4b0 100644 --- a/pipe/l2130i/l2130i_data_source_kafka.yaml +++ b/pipe/l2130i/l2130i_data_source_kafka.yaml @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li191r/li191r_data_source_kafka.yaml b/pipe/li191r/li191r_data_source_kafka.yaml index 6ee6558c2..07461c07c 100644 --- a/pipe/li191r/li191r_data_source_kafka.yaml +++ b/pipe/li191r/li191r_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li191r_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li192sa/li192sa_data_source_kafka.yaml b/pipe/li192sa/li192sa_data_source_kafka.yaml index 19c13a1bf..80fb5c76e 100644 --- a/pipe/li192sa/li192sa_data_source_kafka.yaml +++ b/pipe/li192sa/li192sa_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li192sa_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li7200/li7200_data_source_kafka.yaml b/pipe/li7200/li7200_data_source_kafka.yaml index 3da5dad56..1636969b0 100644 --- a/pipe/li7200/li7200_data_source_kafka.yaml +++ b/pipe/li7200/li7200_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li7200_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li840a/li840a_data_source_kafka.yaml b/pipe/li840a/li840a_data_source_kafka.yaml index a8f924c7e..fe6f8fd53 100644 --- a/pipe/li840a/li840a_data_source_kafka.yaml +++ b/pipe/li840a/li840a_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li840a_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/li850/li850_data_source_kafka.yaml b/pipe/li850/li850_data_source_kafka.yaml index 84aaa28e9..1e4baf716 100644 --- a/pipe/li850/li850_data_source_kafka.yaml +++ b/pipe/li850/li850_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li850_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/mcseries/mcseries_data_source_kafka.yaml b/pipe/mcseries/mcseries_data_source_kafka.yaml index 47b250146..cc987164a 100644 --- a/pipe/mcseries/mcseries_data_source_kafka.yaml +++ b/pipe/mcseries/mcseries_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mcseries_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/metone370380/metone370380_data_source_kafka.yaml b/pipe/metone370380/metone370380_data_source_kafka.yaml index 21c33450f..bb8b8f713 100644 --- a/pipe/metone370380/metone370380_data_source_kafka.yaml +++ b/pipe/metone370380/metone370380_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: metone370380_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml b/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml index 4c260f82e..ec879bf31 100644 --- a/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml +++ b/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mti300ahrs_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/mwseries/mwseries_data_source_kafka.yaml b/pipe/mwseries/mwseries_data_source_kafka.yaml index 4643e0552..19d47de10 100644 --- a/pipe/mwseries/mwseries_data_source_kafka.yaml +++ b/pipe/mwseries/mwseries_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mwseries_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml b/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml index a2188b473..67468c5fc 100644 --- a/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml +++ b/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ncon127tm_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/nr01/nr01_data_source_kafka.yaml b/pipe/nr01/nr01_data_source_kafka.yaml index f4de6c156..564b8d730 100644 --- a/pipe/nr01/nr01_data_source_kafka.yaml +++ b/pipe/nr01/nr01_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: nr01_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/picarro3way/picarro3way_data_source_kafka.yaml b/pipe/picarro3way/picarro3way_data_source_kafka.yaml index cc4a61fbf..14dc581b0 100644 --- a/pipe/picarro3way/picarro3way_data_source_kafka.yaml +++ b/pipe/picarro3way/picarro3way_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: picarro3way_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pluvio/pluvio_data_source_kafka.yaml b/pipe/pluvio/pluvio_data_source_kafka.yaml index a8f927f27..f6bd4c0c3 100644 --- a/pipe/pluvio/pluvio_data_source_kafka.yaml +++ b/pipe/pluvio/pluvio_data_source_kafka.yaml @@ -37,6 +37,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pqs1/pqs1_data_source_kafka.yaml b/pipe/pqs1/pqs1_data_source_kafka.yaml index c33aa8a48..6004b1dc0 100644 --- a/pipe/pqs1/pqs1_data_source_kafka.yaml +++ b/pipe/pqs1/pqs1_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pqs1_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml b/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml index 664c3bf10..c1153f401 100644 --- a/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml +++ b/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pressuretransducer_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/prt/prt_data_source_kafka.yaml b/pipe/prt/prt_data_source_kafka.yaml index 7b575ec30..61613c0e7 100644 --- a/pipe/prt/prt_data_source_kafka.yaml +++ b/pipe/prt/prt_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: prt_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/prtncon/prtncon_data_source_kafka.yaml b/pipe/prtncon/prtncon_data_source_kafka.yaml index 98be52203..19f245fe5 100644 --- a/pipe/prtncon/prtncon_data_source_kafka.yaml +++ b/pipe/prtncon/prtncon_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: prtncon_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/ptb330a/ptb330a_data_source_kafka.yaml b/pipe/ptb330a/ptb330a_data_source_kafka.yaml index 513abca1d..12be61ad4 100644 --- a/pipe/ptb330a/ptb330a_data_source_kafka.yaml +++ b/pipe/ptb330a/ptb330a_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ptb330a_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/pump/pump_data_source_kafka.yaml b/pipe/pump/pump_data_source_kafka.yaml index 585e0e0c0..c5fdf4e3d 100644 --- a/pipe/pump/pump_data_source_kafka.yaml +++ b/pipe/pump/pump_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pump_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/rmyoung/rmyoung_data_source_kafka.yaml b/pipe/rmyoung/rmyoung_data_source_kafka.yaml index 62db5d5a7..ddec22d96 100644 --- a/pipe/rmyoung/rmyoung_data_source_kafka.yaml +++ b/pipe/rmyoung/rmyoung_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: rmyoung_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml b/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml index b72d6e40b..f94144858 100644 --- a/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml +++ b/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml @@ -2,7 +2,7 @@ pipeline: name: ONEOFF_level0_bucket_compact transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 cmd: - sh - "-c" diff --git a/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml b/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml index 69c213087..8ea606760 100644 --- a/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml +++ b/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml @@ -2,7 +2,7 @@ pipeline: name: li7200_parse_kafka_raw_data transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 env: # if use default PARSED_START_INDEX and PARSED_END_INDEX, parse all elements in parse_field # if use default for FIELD_START_INDEX and FIELD_END_INDEX, diff --git a/pipe/si111/si111_data_source_kafka.yaml b/pipe/si111/si111_data_source_kafka.yaml index 26ad86b5b..e0d214d36 100644 --- a/pipe/si111/si111_data_source_kafka.yaml +++ b/pipe/si111/si111_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: si111_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/solenoid/solenoid_data_source_kafka.yaml b/pipe/solenoid/solenoid_data_source_kafka.yaml index 61bc3582f..3f1afa91d 100644 --- a/pipe/solenoid/solenoid_data_source_kafka.yaml +++ b/pipe/solenoid/solenoid_data_source_kafka.yaml @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/spn1/spn1_data_source_kafka.yaml b/pipe/spn1/spn1_data_source_kafka.yaml index ee629b243..bfc0ff44c 100644 --- a/pipe/spn1/spn1_data_source_kafka.yaml +++ b/pipe/spn1/spn1_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: spn1_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/sq500ss/sq500ss_data_source_kafka.yaml b/pipe/sq500ss/sq500ss_data_source_kafka.yaml index 1b67e77b1..ffdbe0081 100644 --- a/pipe/sq500ss/sq500ss_data_source_kafka.yaml +++ b/pipe/sq500ss/sq500ss_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: sq500ss_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/stevenshp/stevenshp_data_source_kafka.yaml b/pipe/stevenshp/stevenshp_data_source_kafka.yaml index 2232e3fae..98dc942ba 100644 --- a/pipe/stevenshp/stevenshp_data_source_kafka.yaml +++ b/pipe/stevenshp/stevenshp_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: stevenshp_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/sunav2/sunav2_data_source_kafka.yaml b/pipe/sunav2/sunav2_data_source_kafka.yaml index 1e1886c43..a563440e7 100644 --- a/pipe/sunav2/sunav2_data_source_kafka.yaml +++ b/pipe/sunav2/sunav2_data_source_kafka.yaml @@ -34,6 +34,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/t7610/t7610_data_source_kafka.yaml b/pipe/t7610/t7610_data_source_kafka.yaml index 1597816d6..3b1c16b16 100644 --- a/pipe/t7610/t7610_data_source_kafka.yaml +++ b/pipe/t7610/t7610_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: t7610_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/tchain/tchain_data_source_kafka.yaml b/pipe/tchain/tchain_data_source_kafka.yaml index d49c6d10f..a8104cca0 100644 --- a/pipe/tchain/tchain_data_source_kafka.yaml +++ b/pipe/tchain/tchain_data_source_kafka.yaml @@ -34,6 +34,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/troll/troll_data_source_kafka.yaml b/pipe/troll/troll_data_source_kafka.yaml index a7f692ccd..a80d4ea82 100644 --- a/pipe/troll/troll_data_source_kafka.yaml +++ b/pipe/troll/troll_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: troll_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/windobserverii/windobserverii_data_source_kafka.yaml b/pipe/windobserverii/windobserverii_data_source_kafka.yaml index 1404986ec..eb0100ba5 100644 --- a/pipe/windobserverii/windobserverii_data_source_kafka.yaml +++ b/pipe/windobserverii/windobserverii_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: windobserverii_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET diff --git a/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml b/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml index 7485da034..bdd84065a 100644 --- a/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml +++ b/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: windobserverii_extreme_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v4.11.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: @@ -26,6 +26,9 @@ transform: - name: pachyderm-kafka-auth env_var: KAFKA_LOG_TOPIC key: KAFKA_LOG_TOPIC + - name: pachyderm-kafka-auth + env_var: KAFKA_COMPACT_TOPIC + key: KAFKA_COMPT_TOPIC - name: l0-bucket env_var: BUCKET_NAME key: LO_BUCKET From 11e3d085049b69f266117633610f19ece461b27b Mon Sep 17 00:00:00 2001 From: Steve Jacobs Date: Mon, 2 Mar 2026 16:35:55 -0500 Subject: [PATCH 2/2] Use 5.1.0 --- modules_combined/kafka_load_and_array_parse/Dockerfile | 2 +- pipe/aepg600m/aepg600m_data_source_kafka.yaml | 2 +- pipe/cmp22/cmp22_data_source_kafka.yaml | 2 +- pipe/csat3/csat3_data_source_kafka.yaml | 2 +- pipe/csat3b/csat3b_data_source_kafka.yaml | 2 +- pipe/drx8533ep/drx8533ep_data_parser.yaml | 2 +- pipe/drx8533ep/drx8533ep_data_source_kafka.yaml | 2 +- pipe/dualfan/dualfan_data_source_kafka.yaml | 2 +- pipe/ecse/ecse_event_data_source_kafka.yaml | 2 +- pipe/exo2/exo2_data_source_kafka.yaml | 2 +- pipe/exo2/exo2_hourly_data_source_kafka.yaml | 2 +- pipe/exo2/exo2_step_data_source_kafka.yaml | 2 +- pipe/g2131i/g2131i_data_source_kafka.yaml | 2 +- pipe/gmp343/gmp343_data_source_kafka.yaml | 2 +- pipe/hfp01sc/hfp01sc_data_source_kafka.yaml | 2 +- pipe/hmp155/hmp155_data_source_kafka.yaml | 2 +- pipe/hmr3300/hmr3300_data_source_kafka.yaml | 2 +- pipe/li191r/li191r_data_source_kafka.yaml | 2 +- pipe/li192sa/li192sa_data_source_kafka.yaml | 2 +- pipe/li7200/li7200_data_source_kafka.yaml | 2 +- pipe/li840a/li840a_data_source_kafka.yaml | 2 +- pipe/li850/li850_data_source_kafka.yaml | 2 +- pipe/mcseries/mcseries_data_source_kafka.yaml | 2 +- pipe/metone370380/metone370380_data_source_kafka.yaml | 2 +- pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml | 2 +- pipe/mwseries/mwseries_data_source_kafka.yaml | 2 +- pipe/ncon127tm/ncon127tm_data_source_kafka.yaml | 2 +- pipe/nr01/nr01_data_source_kafka.yaml | 2 +- pipe/picarro3way/picarro3way_data_source_kafka.yaml | 2 +- pipe/pqs1/pqs1_data_source_kafka.yaml | 2 +- .../pressuretransducer_data_source_kafka.yaml | 2 +- pipe/prt/prt_data_source_kafka.yaml | 2 +- pipe/prtncon/prtncon_data_source_kafka.yaml | 2 +- pipe/ptb330a/ptb330a_data_source_kafka.yaml | 2 +- pipe/pump/pump_data_source_kafka.yaml | 2 +- pipe/rmyoung/rmyoung_data_source_kafka.yaml | 2 +- pipe/sandbox/ONEOFF_level0_bucket_compact.yaml | 2 +- pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml | 2 +- pipe/si111/si111_data_source_kafka.yaml | 2 +- pipe/spn1/spn1_data_source_kafka.yaml | 2 +- pipe/sq500ss/sq500ss_data_source_kafka.yaml | 2 +- pipe/stevenshp/stevenshp_data_source_kafka.yaml | 2 +- pipe/t7610/t7610_data_source_kafka.yaml | 2 +- pipe/troll/troll_data_source_kafka.yaml | 2 +- pipe/windobserverii/windobserverii_data_source_kafka.yaml | 2 +- .../windobserverii_extreme_data_source_kafka.yaml | 2 +- 46 files changed, 46 insertions(+), 46 deletions(-) diff --git a/modules_combined/kafka_load_and_array_parse/Dockerfile b/modules_combined/kafka_load_and_array_parse/Dockerfile index a0821ecf2..669bf187e 100644 --- a/modules_combined/kafka_load_and_array_parse/Dockerfile +++ b/modules_combined/kafka_load_and_array_parse/Dockerfile @@ -5,7 +5,7 @@ # docker build -t neon-is-kfka-load-arry-pars:latest -f ./modules_combined/kafka_load_and_array_parse/Dockerfile . # ### -FROM us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 +FROM us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 ARG MODULE_DIR="modules" ARG APP_DIR="array_parser" diff --git a/pipe/aepg600m/aepg600m_data_source_kafka.yaml b/pipe/aepg600m/aepg600m_data_source_kafka.yaml index d84e00d50..4471e7b4c 100644 --- a/pipe/aepg600m/aepg600m_data_source_kafka.yaml +++ b/pipe/aepg600m/aepg600m_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: aepg600m_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/cmp22/cmp22_data_source_kafka.yaml b/pipe/cmp22/cmp22_data_source_kafka.yaml index 33525eff2..f1659c918 100644 --- a/pipe/cmp22/cmp22_data_source_kafka.yaml +++ b/pipe/cmp22/cmp22_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: cmp22_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/csat3/csat3_data_source_kafka.yaml b/pipe/csat3/csat3_data_source_kafka.yaml index f215b66c2..a8c74561d 100644 --- a/pipe/csat3/csat3_data_source_kafka.yaml +++ b/pipe/csat3/csat3_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: csat3_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/csat3b/csat3b_data_source_kafka.yaml b/pipe/csat3b/csat3b_data_source_kafka.yaml index b2d4781d9..4bd99c132 100644 --- a/pipe/csat3b/csat3b_data_source_kafka.yaml +++ b/pipe/csat3b/csat3b_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: csat3b_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/drx8533ep/drx8533ep_data_parser.yaml b/pipe/drx8533ep/drx8533ep_data_parser.yaml index fd2774ce8..e9cefe008 100644 --- a/pipe/drx8533ep/drx8533ep_data_parser.yaml +++ b/pipe/drx8533ep/drx8533ep_data_parser.yaml @@ -1,7 +1,7 @@ pipeline: name: drx8533ep_data_parser transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 env: # if use default PARSED_START_INDEX and PARSED_END_INDEX, parse all elements in parse_field # if use default for FIELD_START_INDEX and FIELD_END_INDEX, diff --git a/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml b/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml index 0c6ee37fe..19b59028b 100644 --- a/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml +++ b/pipe/drx8533ep/drx8533ep_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: drx8533ep_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/dualfan/dualfan_data_source_kafka.yaml b/pipe/dualfan/dualfan_data_source_kafka.yaml index bcaf0a4d4..379a29b7c 100644 --- a/pipe/dualfan/dualfan_data_source_kafka.yaml +++ b/pipe/dualfan/dualfan_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: dualfan_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/ecse/ecse_event_data_source_kafka.yaml b/pipe/ecse/ecse_event_data_source_kafka.yaml index f7f13b9df..95c5128f1 100644 --- a/pipe/ecse/ecse_event_data_source_kafka.yaml +++ b/pipe/ecse/ecse_event_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ecse_event_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/exo2/exo2_data_source_kafka.yaml b/pipe/exo2/exo2_data_source_kafka.yaml index f6338acda..15e9f12f9 100644 --- a/pipe/exo2/exo2_data_source_kafka.yaml +++ b/pipe/exo2/exo2_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/exo2/exo2_hourly_data_source_kafka.yaml b/pipe/exo2/exo2_hourly_data_source_kafka.yaml index 71237f2c0..782f36eaf 100644 --- a/pipe/exo2/exo2_hourly_data_source_kafka.yaml +++ b/pipe/exo2/exo2_hourly_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_hourly_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/exo2/exo2_step_data_source_kafka.yaml b/pipe/exo2/exo2_step_data_source_kafka.yaml index 067a5046d..e6223be99 100644 --- a/pipe/exo2/exo2_step_data_source_kafka.yaml +++ b/pipe/exo2/exo2_step_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: exo2_step_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/g2131i/g2131i_data_source_kafka.yaml b/pipe/g2131i/g2131i_data_source_kafka.yaml index 7252ab78b..9988b55b7 100644 --- a/pipe/g2131i/g2131i_data_source_kafka.yaml +++ b/pipe/g2131i/g2131i_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: g2131i_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/gmp343/gmp343_data_source_kafka.yaml b/pipe/gmp343/gmp343_data_source_kafka.yaml index f8f1555cb..defa9f941 100644 --- a/pipe/gmp343/gmp343_data_source_kafka.yaml +++ b/pipe/gmp343/gmp343_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: gmp343_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml b/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml index 290e42297..6c04e5aba 100644 --- a/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml +++ b/pipe/hfp01sc/hfp01sc_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hfp01sc_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/hmp155/hmp155_data_source_kafka.yaml b/pipe/hmp155/hmp155_data_source_kafka.yaml index e33720f17..f8a990013 100644 --- a/pipe/hmp155/hmp155_data_source_kafka.yaml +++ b/pipe/hmp155/hmp155_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hmp155_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/hmr3300/hmr3300_data_source_kafka.yaml b/pipe/hmr3300/hmr3300_data_source_kafka.yaml index 466d28deb..0a04dabb5 100644 --- a/pipe/hmr3300/hmr3300_data_source_kafka.yaml +++ b/pipe/hmr3300/hmr3300_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: hmr3300_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/li191r/li191r_data_source_kafka.yaml b/pipe/li191r/li191r_data_source_kafka.yaml index 07461c07c..729dff75f 100644 --- a/pipe/li191r/li191r_data_source_kafka.yaml +++ b/pipe/li191r/li191r_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li191r_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/li192sa/li192sa_data_source_kafka.yaml b/pipe/li192sa/li192sa_data_source_kafka.yaml index 80fb5c76e..a4d86058d 100644 --- a/pipe/li192sa/li192sa_data_source_kafka.yaml +++ b/pipe/li192sa/li192sa_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li192sa_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/li7200/li7200_data_source_kafka.yaml b/pipe/li7200/li7200_data_source_kafka.yaml index 1636969b0..2821b7a3b 100644 --- a/pipe/li7200/li7200_data_source_kafka.yaml +++ b/pipe/li7200/li7200_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li7200_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/li840a/li840a_data_source_kafka.yaml b/pipe/li840a/li840a_data_source_kafka.yaml index fe6f8fd53..6fa6c5209 100644 --- a/pipe/li840a/li840a_data_source_kafka.yaml +++ b/pipe/li840a/li840a_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li840a_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/li850/li850_data_source_kafka.yaml b/pipe/li850/li850_data_source_kafka.yaml index 1e4baf716..4dbf7153a 100644 --- a/pipe/li850/li850_data_source_kafka.yaml +++ b/pipe/li850/li850_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: li850_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/mcseries/mcseries_data_source_kafka.yaml b/pipe/mcseries/mcseries_data_source_kafka.yaml index cc987164a..13eb3cf8a 100644 --- a/pipe/mcseries/mcseries_data_source_kafka.yaml +++ b/pipe/mcseries/mcseries_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mcseries_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/metone370380/metone370380_data_source_kafka.yaml b/pipe/metone370380/metone370380_data_source_kafka.yaml index bb8b8f713..a5b9cad9a 100644 --- a/pipe/metone370380/metone370380_data_source_kafka.yaml +++ b/pipe/metone370380/metone370380_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: metone370380_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml b/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml index ec879bf31..15d124d83 100644 --- a/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml +++ b/pipe/mti300ahrs/mti300ahrs_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mti300ahrs_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/mwseries/mwseries_data_source_kafka.yaml b/pipe/mwseries/mwseries_data_source_kafka.yaml index 19d47de10..3c4f10dea 100644 --- a/pipe/mwseries/mwseries_data_source_kafka.yaml +++ b/pipe/mwseries/mwseries_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: mwseries_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml b/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml index 67468c5fc..3023b72f7 100644 --- a/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml +++ b/pipe/ncon127tm/ncon127tm_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ncon127tm_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/nr01/nr01_data_source_kafka.yaml b/pipe/nr01/nr01_data_source_kafka.yaml index 564b8d730..ba8a70d79 100644 --- a/pipe/nr01/nr01_data_source_kafka.yaml +++ b/pipe/nr01/nr01_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: nr01_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/picarro3way/picarro3way_data_source_kafka.yaml b/pipe/picarro3way/picarro3way_data_source_kafka.yaml index 14dc581b0..d0053dc47 100644 --- a/pipe/picarro3way/picarro3way_data_source_kafka.yaml +++ b/pipe/picarro3way/picarro3way_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: picarro3way_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/pqs1/pqs1_data_source_kafka.yaml b/pipe/pqs1/pqs1_data_source_kafka.yaml index 6004b1dc0..24f7dd8a7 100644 --- a/pipe/pqs1/pqs1_data_source_kafka.yaml +++ b/pipe/pqs1/pqs1_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pqs1_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml b/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml index c1153f401..242bf3ad1 100644 --- a/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml +++ b/pipe/pressuretransducer/pressuretransducer_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pressuretransducer_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/prt/prt_data_source_kafka.yaml b/pipe/prt/prt_data_source_kafka.yaml index 61613c0e7..afc200430 100644 --- a/pipe/prt/prt_data_source_kafka.yaml +++ b/pipe/prt/prt_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: prt_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/prtncon/prtncon_data_source_kafka.yaml b/pipe/prtncon/prtncon_data_source_kafka.yaml index 19f245fe5..19ff6320c 100644 --- a/pipe/prtncon/prtncon_data_source_kafka.yaml +++ b/pipe/prtncon/prtncon_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: prtncon_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/ptb330a/ptb330a_data_source_kafka.yaml b/pipe/ptb330a/ptb330a_data_source_kafka.yaml index 12be61ad4..6d9141e3a 100644 --- a/pipe/ptb330a/ptb330a_data_source_kafka.yaml +++ b/pipe/ptb330a/ptb330a_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: ptb330a_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/pump/pump_data_source_kafka.yaml b/pipe/pump/pump_data_source_kafka.yaml index c5fdf4e3d..99f070110 100644 --- a/pipe/pump/pump_data_source_kafka.yaml +++ b/pipe/pump/pump_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: pump_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/rmyoung/rmyoung_data_source_kafka.yaml b/pipe/rmyoung/rmyoung_data_source_kafka.yaml index ddec22d96..d1261c8b1 100644 --- a/pipe/rmyoung/rmyoung_data_source_kafka.yaml +++ b/pipe/rmyoung/rmyoung_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: rmyoung_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml b/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml index f94144858..187a1593a 100644 --- a/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml +++ b/pipe/sandbox/ONEOFF_level0_bucket_compact.yaml @@ -2,7 +2,7 @@ pipeline: name: ONEOFF_level0_bucket_compact transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 cmd: - sh - "-c" diff --git a/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml b/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml index 8ea606760..c12f6990a 100644 --- a/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml +++ b/pipe/sandbox/li7200_parse_kafka_raw_data_errCmd.yaml @@ -2,7 +2,7 @@ pipeline: name: li7200_parse_kafka_raw_data transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 env: # if use default PARSED_START_INDEX and PARSED_END_INDEX, parse all elements in parse_field # if use default for FIELD_START_INDEX and FIELD_END_INDEX, diff --git a/pipe/si111/si111_data_source_kafka.yaml b/pipe/si111/si111_data_source_kafka.yaml index e0d214d36..eba3e322b 100644 --- a/pipe/si111/si111_data_source_kafka.yaml +++ b/pipe/si111/si111_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: si111_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/spn1/spn1_data_source_kafka.yaml b/pipe/spn1/spn1_data_source_kafka.yaml index bfc0ff44c..b4040c725 100644 --- a/pipe/spn1/spn1_data_source_kafka.yaml +++ b/pipe/spn1/spn1_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: spn1_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/sq500ss/sq500ss_data_source_kafka.yaml b/pipe/sq500ss/sq500ss_data_source_kafka.yaml index ffdbe0081..cff939d5a 100644 --- a/pipe/sq500ss/sq500ss_data_source_kafka.yaml +++ b/pipe/sq500ss/sq500ss_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: sq500ss_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/stevenshp/stevenshp_data_source_kafka.yaml b/pipe/stevenshp/stevenshp_data_source_kafka.yaml index 98dc942ba..1e952c1f8 100644 --- a/pipe/stevenshp/stevenshp_data_source_kafka.yaml +++ b/pipe/stevenshp/stevenshp_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: stevenshp_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/t7610/t7610_data_source_kafka.yaml b/pipe/t7610/t7610_data_source_kafka.yaml index 3b1c16b16..a63dad128 100644 --- a/pipe/t7610/t7610_data_source_kafka.yaml +++ b/pipe/t7610/t7610_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: t7610_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/troll/troll_data_source_kafka.yaml b/pipe/troll/troll_data_source_kafka.yaml index a80d4ea82..8a81a611c 100644 --- a/pipe/troll/troll_data_source_kafka.yaml +++ b/pipe/troll/troll_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: troll_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/windobserverii/windobserverii_data_source_kafka.yaml b/pipe/windobserverii/windobserverii_data_source_kafka.yaml index eb0100ba5..f89545395 100644 --- a/pipe/windobserverii/windobserverii_data_source_kafka.yaml +++ b/pipe/windobserverii/windobserverii_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: windobserverii_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml b/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml index bdd84065a..ddc9c00f7 100644 --- a/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml +++ b/pipe/windobserverii_extreme/windobserverii_extreme_data_source_kafka.yaml @@ -2,7 +2,7 @@ pipeline: name: windobserverii_extreme_data_source_kafka transform: - image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.0.0 + image: us-central1-docker.pkg.dev/neon-shared-service/bei/neon-avro-kafka-loader:v5.1.0 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: