From 30dabd2dd826791506411f14847b6f9ca85ec918 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 2 May 2025 16:03:54 +0000 Subject: [PATCH 1/8] Remove unwanted files --- main.sh | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/main.sh b/main.sh index 4a52e7a..054aae3 100755 --- a/main.sh +++ b/main.sh @@ -802,7 +802,8 @@ append_azure_blob_output_block() { local path=$2 local container_name=${3:-system} # Default to 'system' if not provided - cat >> ./fluent-bit.conf << EOF + if [[ -n "${AZURE_STORAGE_AUTH_INTEGRATION_ID}" ]]; then + cat >> ./fluent-bit.conf << EOF [OUTPUT] Name azure_blob @@ -815,6 +816,21 @@ append_azure_blob_output_block() { auto_create_container on tls on EOF + else + cat >> ./fluent-bit.conf << EOF + +[OUTPUT] + Name azure_blob + Match ${match} + account_name ${STORAGE_ACCOUNT_NAME} + shared_key ${SHARED_KEY} + blob_type blockblob + path ${path} + container_name ${container_name} + auto_create_container on + tls on +EOF + fi } #}}}: append_azure_blob_output_block @@ -1059,6 +1075,7 @@ fetch_organization_info() { #{{{ SG_RUNNER_GROUP_SIGNATURE="$(echo "${response}" | jq -r '.data.RunnerGroup.RunnerGroupSignature // empty')" # TAGS="$(echo "${response}" | jq -r '.data.Tags')" STORAGE_ACCOUNT_NAME="$(echo "${response}" | jq -r '.data.RunnerGroup.StorageBackendConfig.azureBlobStorageAccountName // empty')" + AZURE_STORAGE_AUTH_INTEGRATION_ID="$( echo "${response}" | jq -r '.data.RunnerGroup.StorageBackendConfig.auth.integrationId // empty')" SHARED_KEY="$(echo "${response}" | jq -r '.data.RunnerGroup.StorageBackendConfig.azureBlobStorageAccessKey // empty')" STORAGE_BACKEND_TYPE="$(echo "${response}" | jq -r '.data.RunnerGroup.StorageBackendConfig.type // empty')" S3_BUCKET_NAME="$(echo "${response}" | jq -r '.data.RunnerGroup.StorageBackendConfig.s3BucketName // empty')" @@ -1081,7 +1098,13 @@ fetch_organization_info() { #{{{ exit 1 fi elif [[ "$STORAGE_BACKEND_TYPE" == "azure_blob_storage" ]]; then - for var in SHARED_KEY STORAGE_ACCOUNT_NAME; do + keys=("STORAGE_ACCOUNT_NAME") + if [[ -n "${AZURE_STORAGE_AUTH_INTEGRATION_ID}" ]]; then + keys+=("AZURE_STORAGE_AUTH_INTEGRATION_ID") + else + keys+=("SHARED_KEY") + fi + for var in "${keys}"; do check_variable_value "$var" done else From 35a29fc6df9d0571c1c934e63084832c712d0743 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 2 May 2025 16:15:00 +0000 Subject: [PATCH 2/8] revert unwanted changes --- main.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/main.sh b/main.sh index 054aae3..03865e3 100755 --- a/main.sh +++ b/main.sh @@ -944,7 +944,8 @@ elif [[ "${STORAGE_BACKEND_TYPE}" == "azure_blob_storage" ]]; then append_azure_blob_output_block "fluentbit" "fluentbit/log" append_azure_blob_output_block "ecsagent" "ecsagent/log" append_azure_blob_output_block "registrationinfo" "registrationinfo/log" - cat >> ./fluent-bit.conf << EOF + if [[ -n "${AZURE_STORAGE_AUTH_INTEGRATION_ID}" ]]; then + cat >> ./fluent-bit.conf << EOF [OUTPUT] Name azure_blob @@ -955,6 +956,18 @@ elif [[ "${STORAGE_BACKEND_TYPE}" == "azure_blob_storage" ]]; then auto_create_container on tls on EOF + else + cat >> ./fluent-bit.conf << EOF +[OUTPUT] + Name azure_blob + Match_Regex orgs** + account_name ${STORAGE_ACCOUNT_NAME} + shared_key ${SHARED_KEY} + container_name runner + auto_create_container on + tls on +EOF + fi fi spinner_msg "Configuring local data" 0 From cf3c3a129f38d9a5bcb69f9059a751dfe83c52ef Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 May 2025 13:05:09 +0000 Subject: [PATCH 3/8] changes for sgrunner runner service --- main.sh | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/main.sh b/main.sh index 03865e3..c8a612a 100755 --- a/main.sh +++ b/main.sh @@ -673,6 +673,8 @@ spinner() { #{{{ clean_local_setup() { #{{{ debug "Stopping services.." + systemctl stop sgrunner 2>/dev/null + debug "Stopping sgrunner.." systemctl stop ecs 2>/dev/null debug "Stopping $CONTAINER_ORCHESTRATOR containers.." $CONTAINER_ORCHESTRATOR stop ecs-agent fluentbit-agent >&/dev/null @@ -697,6 +699,9 @@ clean_local_setup() { #{{{ "/etc/systemd/system/ecs.service.d/http-proxy.conf" "/etc/systemd/system/amazon-ssm-agent.service.d/http-proxy.conf" "/etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/http-proxy.conf" + "/var/log/sgrunner" + "/etc/systemd/system/sgrunner.service" + "/opt/sgrunner" ) # Loop through the array and remove each item @@ -1253,6 +1258,7 @@ register_instance() { #{{{ fetch_organization_info configure_local_data + configure_golang_service configure_fluentbit configure_local_network @@ -1651,6 +1657,54 @@ EOF fi } +configure_golang_service(){ + info "Configuring golang service" + + mkdir -p /opt/sgrunner/ + cp sgrunner /opt/sgrunner/sgrunner + + mkdir -p /var/log/sgrunner/ + mkdir -p /etc/sgrunner + debug "writing sgrunner config" + cat > /etc/sgrunner/config.yaml << EOF +sg_org: ${ORGANIZATION_NAME} +sg_runner_group: ${RUNNER_GROUP_ID} +sg_runner_group_token: ${SG_NODE_TOKEN} +integration_id: ${AZURE_STORAGE_AUTH_INTEGRATION_ID} +log_level: info +EOF + debug "generated configuration for golang service" + + debug "writing sgrunner systemd file" + cat > /etc/systemd/system/sgrunner.service << EOF +[Unit] +Description=My Go Application +After=network.target + +[Service] +Type=simple +ExecStart=/opt/sgrunner/sgrunner +WorkingDirectory=/opt/sgrunner +Restart=on-failure +RestartSec=5s + +# Logging +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target +EOF + debug "generated systemd file for sgrunner service" + + systemctl daemon-reload + systemctl start sgrunner + if [[ $? != 0 ]]; then + err "failed to start the sgrunner service" + exit 1 + fi +} + main() { #{{{ [[ "${*}" =~ --help || $# -lt 1 ]] && show_help && exit 0 From 2bccc6f23c9cfa12a663f0b89ad568d31083946c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 8 May 2025 14:59:40 +0000 Subject: [PATCH 4/8] Add logrotate configuration --- main.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/main.sh b/main.sh index c8a612a..1a946e0 100755 --- a/main.sh +++ b/main.sh @@ -1660,6 +1660,20 @@ EOF configure_golang_service(){ info "Configuring golang service" + # Create logrotate file + cat > /etc/logrotate.d/sgrunner < Date: Mon, 19 May 2025 10:38:24 +0000 Subject: [PATCH 5/8] fix: hard coded private address and dynamic port for golang service --- main.sh | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/main.sh b/main.sh index 1a946e0..ea89eb0 100755 --- a/main.sh +++ b/main.sh @@ -6,6 +6,8 @@ set -o pipefail #{{{ Environment variables +readonly PRIVATE_IP_ADDRESS="$(ip route | grep default | cut -d" " -f9)" + ## main CONTAINER_ORCHESTRATOR= LOG_DEBUG=${LOG_DEBUG:=false} @@ -811,15 +813,19 @@ append_azure_blob_output_block() { cat >> ./fluent-bit.conf << EOF [OUTPUT] - Name azure_blob - Match ${match} - account_name ${STORAGE_ACCOUNT_NAME} - shared_key ${SHARED_KEY} - blob_type blockblob - path ${path} - container_name ${container_name} - auto_create_container on - tls on + Name http + Match ${match} + Host ${PRIVATE_IP_ADDRESS} + Port 49153 + URI /fluentbit/log + Format json + json_date_format iso8601 + json_date_key @timestamp + header path ${path} + header blob_type blockblob + header container_name ${container_name} + header account_name ${STORAGE_ACCOUNT_NAME} + header_tag fluent_tag EOF else cat >> ./fluent-bit.conf << EOF @@ -955,11 +961,16 @@ elif [[ "${STORAGE_BACKEND_TYPE}" == "azure_blob_storage" ]]; then [OUTPUT] Name azure_blob Match_Regex orgs** - account_name ${STORAGE_ACCOUNT_NAME} - shared_key ${SHARED_KEY} - container_name runner - auto_create_container on - tls on + Host ${PRIVATE_IP_ADDRESS} + Port 49153 + URI /fluentbit/log + Format json + json_date_format iso8601 + json_date_key @timestamp + header blob_type appendblob + header account_name ${STORAGE_ACCOUNT_NAME} + header container_name runner + header_tag fluent_tag EOF else cat >> ./fluent-bit.conf << EOF @@ -1692,7 +1703,7 @@ EOF debug "writing sgrunner systemd file" cat > /etc/systemd/system/sgrunner.service << EOF [Unit] -Description=My Go Application +Description=Stackguardian Runner After=network.target [Service] From 38f519d93fc028c6a106607da4a50f96e70e7793 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 22 May 2025 09:18:36 +0000 Subject: [PATCH 6/8] bug fixes --- main.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/main.sh b/main.sh index ea89eb0..c63525d 100755 --- a/main.sh +++ b/main.sh @@ -701,7 +701,6 @@ clean_local_setup() { #{{{ "/etc/systemd/system/ecs.service.d/http-proxy.conf" "/etc/systemd/system/amazon-ssm-agent.service.d/http-proxy.conf" "/etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/http-proxy.conf" - "/var/log/sgrunner" "/etc/systemd/system/sgrunner.service" "/opt/sgrunner" ) @@ -1647,7 +1646,7 @@ EOF # sets proxy configuration for containers # Required by fluentbit mkdir -p "${HOME}/.docker" - http_proxy_docker_config="{ \"proxies\": { \"default\": { \"httpProxy\": \"http://${HTTP_PROXY}\", \"httpsProxy\": \"http://${HTTP_PROXY}\", \"noProxy\": \"${NO_PROXY}\" } } }" + http_proxy_docker_config="{ \"proxies\": { \"default\": { \"httpProxy\": \"http://${HTTP_PROXY}\", \"httpsProxy\": \"http://${HTTP_PROXY}\", \"noProxy\": \"${PRIVATE_IP_ADDRESS},${NO_PROXY}\" } } }" [[ -e "$HOME/.docker/config.json" ]] && cp "$HOME/.docker/config.json" "$HOME/original_docker_config.json" patch_json "$HOME/.docker/config.json" "$http_proxy_docker_config" @@ -1686,7 +1685,7 @@ configure_golang_service(){ EOF mkdir -p /opt/sgrunner/ - cp sgrunner /opt/sgrunner/sgrunner + cp sgrunner /opt/sgrunner/sgrunner 2>/dev/null mkdir -p /var/log/sgrunner/ mkdir -p /etc/sgrunner @@ -1695,9 +1694,12 @@ EOF sg_org: ${ORGANIZATION_NAME} sg_runner_group: ${RUNNER_GROUP_ID} sg_runner_group_token: ${SG_NODE_TOKEN} -integration_id: ${AZURE_STORAGE_AUTH_INTEGRATION_ID} -log_level: info +sg_integration_id: ${AZURE_STORAGE_AUTH_INTEGRATION_ID} +sg_log_level: info EOF + if [[ -n "${HTTP_PROXY}" ]]; then + echo "sg_proxy_address: ${HTTP_PROXY}" >> /etc/sgrunner/config.yaml + fi debug "generated configuration for golang service" debug "writing sgrunner systemd file" @@ -1724,7 +1726,7 @@ EOF systemctl daemon-reload systemctl start sgrunner - if [[ $? != 0 ]]; then + if ! check_systemctl_status "sgrunner" ; then err "failed to start the sgrunner service" exit 1 fi From 7fc5096963195816c78ee506faedeba9c4836bfc Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 22 May 2025 12:17:17 +0000 Subject: [PATCH 7/8] fix: incorrect json while sending error message in runner group --- main.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.sh b/main.sh index c63525d..adb553b 100755 --- a/main.sh +++ b/main.sh @@ -343,7 +343,7 @@ check_fluentbit_status() { #{{{ if ignore_fluentbit_errors; then debug "Ignoring Fluentbit error(s) $err_msg" else - err "Fluentbit encountered error(s)" "$err_msg" + err "Fluentbit encountered error(s)" "failed to start fluentbit" if ! no_clean_on_fail; then clean_local_setup & spinner "$!" "Starting cleanup" info "Use --no-clean-on-fail to not clean up after Fluentbit errors are encountered for debugging issues" @@ -496,7 +496,7 @@ cgroupsv2() { #{{{ #}}}: cgroupsv2 update_runner_group(){ - url="${SG_BASE_API}/orgs/${ORGANIZATION_ID}/runnergroups/${RUNNER_GROUP_ID}/" + url="${SG_BASE_API}/orgs/${ORGANIZATION_NAME}/runnergroups/${RUNNER_GROUP_ID}/" err_msg=$(echo -n "$1" | tr -cd "[:print:]") From 211f2b0d56790a75be40da07dc6088901fa98b3e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 26 May 2025 07:48:20 +0000 Subject: [PATCH 8/8] bug fix: output to http from azure --- main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.sh b/main.sh index adb553b..e0b69c4 100755 --- a/main.sh +++ b/main.sh @@ -958,7 +958,7 @@ elif [[ "${STORAGE_BACKEND_TYPE}" == "azure_blob_storage" ]]; then cat >> ./fluent-bit.conf << EOF [OUTPUT] - Name azure_blob + Name http Match_Regex orgs** Host ${PRIVATE_IP_ADDRESS} Port 49153