From af3ace2357bbd7f7356e345d309399f10cbb1093 Mon Sep 17 00:00:00 2001 From: Prince Datta Date: Tue, 17 Sep 2024 13:47:04 +0530 Subject: [PATCH 1/7] dummy --- BUILD | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/BUILD b/BUILD index c8efdeac0..291cd9530 100644 --- a/BUILD +++ b/BUILD @@ -6,7 +6,7 @@ test_suite( ":test_cloud_sql_proxy", ":test_dr_elephant", ":test_hive_hcatalog", - ":test_hive_llap", +# ":test_hive_llap", ":test_starburst_presto", ":test_spark_rapids", "//alluxio:test_alluxio", @@ -21,20 +21,20 @@ test_suite( "//gpu:test_gpu", "//h2o:test_h2o", "//hbase:test_hbase", - "//horovod:test_horovod", +# "//horovod:test_horovod", "//hue:test_hue", "//kafka:test_kafka", "//knox:test_knox", "//livy:test_livy", - "//mlvm:test_mlvm", +# "//mlvm:test_mlvm", "//oozie:test_oozie", "//presto:test_presto", "//ranger:test_ranger", - "//rapids:test_rapids", +# "//rapids:test_rapids", "//rstudio:test_rstudio", "//solr:test_solr", "//sqoop:test_sqoop", - "//tony:test_tony", +# "//tony:test_tony", ], ) From 9867d539e4d1d2626d6d9a8e43c7c574a12a4c2d Mon Sep 17 00:00:00 2001 From: Prince Datta Date: Wed, 18 Sep 2024 10:30:26 +0530 Subject: [PATCH 2/7] dummy --- integration_tests/dataproc_test_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dataproc_test_case.py b/integration_tests/dataproc_test_case.py index 6f2bd9f28..436b816d6 100644 --- a/integration_tests/dataproc_test_case.py +++ b/integration_tests/dataproc_test_case.py @@ -117,7 +117,7 @@ def createCluster(self, args.append("--image-version={}".format(FLAGS.image_version)) if FLAGS.image_version in self.IMAGE_VERSION_2_2: args.append("--public-ip-address") - + #dummy for i in init_actions: if "install_gpu_driver.sh" in i or \ "mlvm.sh" in i or "rapids.sh" in i or \ From 50fb601e80533e1ac49798f5bcc312143c731539 Mon Sep 17 00:00:00 2001 From: Prince Datta Date: Thu, 19 Sep 2024 11:02:14 +0530 Subject: [PATCH 3/7] dummy --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 291cd9530..a314277f9 100644 --- a/BUILD +++ b/BUILD @@ -11,7 +11,7 @@ test_suite( ":test_spark_rapids", "//alluxio:test_alluxio", "//atlas:test_atlas", - "//bigtable:test_bigtable", +# "//bigtable:test_bigtable", "//conda:test_conda", "//connectors:test_connectors", "//dask:test_dask", From 87c7787e69cd2e50d1bb4896b1e464d9e88dbcef Mon Sep 17 00:00:00 2001 From: Prince Datta Date: Thu, 19 Sep 2024 12:46:55 +0530 Subject: [PATCH 4/7] dummy --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index a314277f9..ace9681c2 100644 --- a/BUILD +++ b/BUILD @@ -8,7 +8,7 @@ test_suite( ":test_hive_hcatalog", # ":test_hive_llap", ":test_starburst_presto", - ":test_spark_rapids", +# ":test_spark_rapids", "//alluxio:test_alluxio", "//atlas:test_atlas", # "//bigtable:test_bigtable", From 5c0a606d468c78290edb51ed3656e5db63688a5e Mon Sep 17 00:00:00 2001 From: Prince Datta Date: Wed, 2 Oct 2024 18:46:03 +0530 Subject: [PATCH 5/7] dummy --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index ace9681c2..17427426f 100644 --- a/BUILD +++ b/BUILD @@ -18,7 +18,7 @@ test_suite( "//drill:test_drill", "//flink:test_flink", "//ganglia:test_ganglia", - "//gpu:test_gpu", +# "//gpu:test_gpu", "//h2o:test_h2o", "//hbase:test_hbase", # "//horovod:test_horovod", From e2ed0b458957543ea4991433ad77ab57318282fc Mon Sep 17 00:00:00 2001 From: Prince Datta Date: Thu, 3 Oct 2024 13:08:39 +0530 Subject: [PATCH 6/7] pinning libsystemd0 version for debian 12 OS --- cloud-sql-proxy/cloud-sql-proxy.sh | 17 +++++++++-- kafka/kafka-manager.sh | 11 +++++++- rstudio/rstudio.sh | 45 ++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/cloud-sql-proxy/cloud-sql-proxy.sh b/cloud-sql-proxy/cloud-sql-proxy.sh index cd7961df2..ccb1a4c28 100755 --- a/cloud-sql-proxy/cloud-sql-proxy.sh +++ b/cloud-sql-proxy/cloud-sql-proxy.sh @@ -472,6 +472,11 @@ function run_validation() { log 'Validated Hive functioning' } +function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs ; } +function os_version() { grep '^VERSION_ID=' /etc/os-release | cut -d= -f2 | xargs ; } +function is_debian() { [[ "$(os_id)" == 'debian' ]] ; } +function is_debian12() { is_debian && [[ "$(os_version)" == '12'* ]] ; } + function install_mysql_cli() { if command -v mysql >/dev/null; then log "MySQL CLI is already installed" @@ -480,7 +485,11 @@ function install_mysql_cli() { log "Installing MySQL CLI ..." if command -v apt >/dev/null; then - apt update && apt install mysql-client -y + if is_debian12; then + apt update && apt install libsystemd0=252.26-1~deb12u2 mysql-client -y + else + apt update && apt install mysql-client -y + fi elif command -v yum >/dev/null; then yum -y update && yum -y install mysql fi @@ -495,7 +504,11 @@ function install_postgres_cli() { log "Installing POSTGRES CLI ..." if command -v apt >/dev/null; then - apt update && apt install postgresql-client -y + if is_debian12; then + apt update && apt install libsystemd0=252.26-1~deb12u2 postgresql-client -y + else + apt update && apt install postgresql-client -y + fi elif command -v yum >/dev/null; then yum -y update && yum -y install postgresql fi diff --git a/kafka/kafka-manager.sh b/kafka/kafka-manager.sh index 6f7766d4e..22a8307d4 100755 --- a/kafka/kafka-manager.sh +++ b/kafka/kafka-manager.sh @@ -20,6 +20,11 @@ set -euxo pipefail +function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs ; } +function os_version() { grep '^VERSION_ID=' /etc/os-release | cut -d= -f2 | xargs ; } +function is_debian() { [[ "$(os_id)" == 'debian' ]] ; } +function is_debian12() { is_debian && [[ "$(os_version)" == '12'* ]] ; } + readonly KAFKA_MANAGER_HTTP_PORT="$(/usr/share/google/get_metadata_value attributes/kafka-manager-http-port || echo 9000)" readonly KAFKA_MANAGER_GIT_URI=https://github.com/yahoo/CMAK.git readonly KAFKA_MANAGER_GIT_DIR=/tmp/kafka-manager @@ -29,7 +34,11 @@ readonly ZOOKEEPER_CONFIG=/etc/zookeeper/conf/zoo.cfg function install_packages(){ apt-get update - apt-get install -yq apt-transport-https curl gnupg + if is_debian12; then + apt-get install -yq libsystemd0=252.26-1~deb12u2 apt-transport-https curl gnupg + else + apt-get install -yq apt-transport-https curl gnupg + fi } function add_sources(){ diff --git a/rstudio/rstudio.sh b/rstudio/rstudio.sh index a321a22e7..bddbd159d 100755 --- a/rstudio/rstudio.sh +++ b/rstudio/rstudio.sh @@ -17,6 +17,13 @@ set -euxo pipefail +function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs ; } +function os_version() { grep '^VERSION_ID=' /etc/os-release | cut -d= -f2 | xargs ; } +function is_debian() { [[ "$(os_id)" == 'debian' ]] ; } +function is_debian12() { is_debian && [[ "$(os_version)" == '12'* ]] ; } +function is_ubuntu() { [[ "$(os_id)" == 'ubuntu' ]] ; } +function is_ubuntu18() { is_ubuntu && [[ "$(os_version)" == '18.04'* ]] ; } + # Only run on the master node ROLE="$(/usr/share/google/get_metadata_value attributes/dataproc-role)" @@ -103,15 +110,25 @@ function run_with_retries() { "${cmd[@]}" } -function install_package() { - local LIBDEFLATE0_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate0_1.5-3_amd64.deb" - local LIBDEFLATE_DEV_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate-dev_1.5-3_amd64.deb" - TMP_DIR=$(mktemp -d) - wget -q -P "${TMP_DIR}" "${LIBDEFLATE0_URL}" - dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE0_URL}")" - wget -q -P "${TMP_DIR}" "${LIBDEFLATE_DEV_URL}" - dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE_DEV_URL}")" - rm -rf "${TMP_DIR}" +function install_libflate_package() { + if is_ubuntu18; then + local LIBDEFLATE0_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate0_1.5-3_amd64.deb" + local LIBDEFLATE_DEV_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate-dev_1.5-3_amd64.deb" + TMP_DIR=$(mktemp -d) + wget -q -P "${TMP_DIR}" "${LIBDEFLATE0_URL}" + dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE0_URL}")" + wget -q -P "${TMP_DIR}" "${LIBDEFLATE_DEV_URL}" + dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE_DEV_URL}")" + rm -rf "${TMP_DIR}" + fi +} + +function install_r_packages() { + if is_debian12; then + apt-get install -y libsystemd0=252.26-1~deb12u2 r-base r-base-dev gdebi-core + else + apt-get install -y r-base r-base-dev gdebi-core + fi } if [[ "${ROLE}" == 'Master' ]]; then @@ -138,17 +155,15 @@ if [[ "${ROLE}" == 'Master' ]]; then if [[ "${OS_ID}" == "ubuntu" ]]; then curl -fsSL --retry-connrefused --retry 10 --retry-max-time 30 https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc fi - if [[ ${OS_ID} == debian ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION}" == "2.2" | bc -l) == 1 ]]; then - apt-get install -y software-properties-common libsystemd0=252.26-1~deb12u2 + if is_debian12; then + apt-get install -y libsystemd0=252.26-1~deb12u2 software-properties-common else apt-get install -y software-properties-common fi add-apt-repository "deb http://cran.r-project.org/bin/linux/${OS_ID} ${OS_CODE}-cran40/" - if [[ ${OS_ID} == ubuntu ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} < 2.1" | bc -l) == 1 ]]; then - install_package - fi + install_libflate_package update_apt_get - apt-get install -y r-base r-base-dev gdebi-core + install_r_packages # Download and install RStudio Server package: wget -nv --timeout=30 --tries=5 --retry-connrefused ${RSTUDIO_SERVER_PACKAGE_URI} -P /tmp From d2fe646a8235f9eb70dfd6885987b4a7c10d025a Mon Sep 17 00:00:00 2001 From: Prince Datta Date: Mon, 7 Oct 2024 13:31:26 +0530 Subject: [PATCH 7/7] dummy --- BUILD | 2 +- cloud-sql-proxy/cloud-sql-proxy.sh | 17 ++--------------- kafka/kafka-manager.sh | 11 +---------- rstudio/rstudio.sh | 12 +----------- 4 files changed, 5 insertions(+), 37 deletions(-) diff --git a/BUILD b/BUILD index 17427426f..a5567f1b9 100644 --- a/BUILD +++ b/BUILD @@ -11,7 +11,7 @@ test_suite( # ":test_spark_rapids", "//alluxio:test_alluxio", "//atlas:test_atlas", -# "//bigtable:test_bigtable", + "//bigtable:test_bigtable", "//conda:test_conda", "//connectors:test_connectors", "//dask:test_dask", diff --git a/cloud-sql-proxy/cloud-sql-proxy.sh b/cloud-sql-proxy/cloud-sql-proxy.sh index ccb1a4c28..cd7961df2 100755 --- a/cloud-sql-proxy/cloud-sql-proxy.sh +++ b/cloud-sql-proxy/cloud-sql-proxy.sh @@ -472,11 +472,6 @@ function run_validation() { log 'Validated Hive functioning' } -function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs ; } -function os_version() { grep '^VERSION_ID=' /etc/os-release | cut -d= -f2 | xargs ; } -function is_debian() { [[ "$(os_id)" == 'debian' ]] ; } -function is_debian12() { is_debian && [[ "$(os_version)" == '12'* ]] ; } - function install_mysql_cli() { if command -v mysql >/dev/null; then log "MySQL CLI is already installed" @@ -485,11 +480,7 @@ function install_mysql_cli() { log "Installing MySQL CLI ..." if command -v apt >/dev/null; then - if is_debian12; then - apt update && apt install libsystemd0=252.26-1~deb12u2 mysql-client -y - else - apt update && apt install mysql-client -y - fi + apt update && apt install mysql-client -y elif command -v yum >/dev/null; then yum -y update && yum -y install mysql fi @@ -504,11 +495,7 @@ function install_postgres_cli() { log "Installing POSTGRES CLI ..." if command -v apt >/dev/null; then - if is_debian12; then - apt update && apt install libsystemd0=252.26-1~deb12u2 postgresql-client -y - else - apt update && apt install postgresql-client -y - fi + apt update && apt install postgresql-client -y elif command -v yum >/dev/null; then yum -y update && yum -y install postgresql fi diff --git a/kafka/kafka-manager.sh b/kafka/kafka-manager.sh index 22a8307d4..6f7766d4e 100755 --- a/kafka/kafka-manager.sh +++ b/kafka/kafka-manager.sh @@ -20,11 +20,6 @@ set -euxo pipefail -function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs ; } -function os_version() { grep '^VERSION_ID=' /etc/os-release | cut -d= -f2 | xargs ; } -function is_debian() { [[ "$(os_id)" == 'debian' ]] ; } -function is_debian12() { is_debian && [[ "$(os_version)" == '12'* ]] ; } - readonly KAFKA_MANAGER_HTTP_PORT="$(/usr/share/google/get_metadata_value attributes/kafka-manager-http-port || echo 9000)" readonly KAFKA_MANAGER_GIT_URI=https://github.com/yahoo/CMAK.git readonly KAFKA_MANAGER_GIT_DIR=/tmp/kafka-manager @@ -34,11 +29,7 @@ readonly ZOOKEEPER_CONFIG=/etc/zookeeper/conf/zoo.cfg function install_packages(){ apt-get update - if is_debian12; then - apt-get install -yq libsystemd0=252.26-1~deb12u2 apt-transport-https curl gnupg - else - apt-get install -yq apt-transport-https curl gnupg - fi + apt-get install -yq apt-transport-https curl gnupg } function add_sources(){ diff --git a/rstudio/rstudio.sh b/rstudio/rstudio.sh index bddbd159d..a3602f1aa 100755 --- a/rstudio/rstudio.sh +++ b/rstudio/rstudio.sh @@ -19,8 +19,6 @@ set -euxo pipefail function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs ; } function os_version() { grep '^VERSION_ID=' /etc/os-release | cut -d= -f2 | xargs ; } -function is_debian() { [[ "$(os_id)" == 'debian' ]] ; } -function is_debian12() { is_debian && [[ "$(os_version)" == '12'* ]] ; } function is_ubuntu() { [[ "$(os_id)" == 'ubuntu' ]] ; } function is_ubuntu18() { is_ubuntu && [[ "$(os_version)" == '18.04'* ]] ; } @@ -123,14 +121,6 @@ function install_libflate_package() { fi } -function install_r_packages() { - if is_debian12; then - apt-get install -y libsystemd0=252.26-1~deb12u2 r-base r-base-dev gdebi-core - else - apt-get install -y r-base r-base-dev gdebi-core - fi -} - if [[ "${ROLE}" == 'Master' ]]; then if [[ ${OS_ID} == debian ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} <= 2.1" | bc -l) == 1 ]]; then remove_old_backports @@ -163,7 +153,7 @@ if [[ "${ROLE}" == 'Master' ]]; then add-apt-repository "deb http://cran.r-project.org/bin/linux/${OS_ID} ${OS_CODE}-cran40/" install_libflate_package update_apt_get - install_r_packages + apt-get install -y r-base r-base-dev gdebi-core # Download and install RStudio Server package: wget -nv --timeout=30 --tries=5 --retry-connrefused ${RSTUDIO_SERVER_PACKAGE_URI} -P /tmp