From 7fddaa3222a6466bd76f644a137eede4f31e8376 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Mon, 2 Jun 2025 16:19:39 +0000 Subject: [PATCH 01/28] Starting to add in db. --- .../backstage/backstage-quickstart/db.tf | 13 +++++++++++++ .../manifests/app-config-production.yaml | 12 ++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/app-config-production.yaml diff --git a/reference-architectures/backstage/backstage-quickstart/db.tf b/reference-architectures/backstage/backstage-quickstart/db.tf index f73615e8..0267a2f0 100644 --- a/reference-architectures/backstage/backstage-quickstart/db.tf +++ b/reference-architectures/backstage/backstage-quickstart/db.tf @@ -31,6 +31,7 @@ resource "google_sql_database_instance" "instance" { psc_enabled = true allowed_consumer_projects = [var.environment_project_id] } + ssl_mode = "ENCRYPTED_ONLY" } } timeouts { @@ -38,9 +39,21 @@ resource "google_sql_database_instance" "instance" { update = "30m" delete = "30m" } +} +resource "google_sql_database" "database" { + name = "backstage" + instance = google_sql_database_instance.instance.name } +resource "google_sql_user" "iam_service_account_user" { + # Note: for Postgres only, GCP requires omitting the ".gserviceaccount.com" suffix + # from the service account email due to length limits on database usernames. + + name = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") + instance = google_sql_database_instance.instance.name + type = "CLOUD_IAM_SERVICE_ACCOUNT" +} resource "null_resource" "sqlIamDelay" { provisioner "local-exec" { diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/app-config-production.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/app-config-production.yaml new file mode 100644 index 00000000..936bf0de --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/app-config-production.yaml @@ -0,0 +1,12 @@ +database: + client: pg + connection: + host: ${POSTGRES_HOST} + port: ${POSTGRES_PORT} + database: ${POSTGRES_DB} + #user: backstage-qs-workload@backstage-453421.iam[this is just to show that the IAM account is trimmed when we authenticate to the cloudsql via IAM] + user: ${POSTGRES_USER} + ssl: + rejectUnauthorized: false +options: +requestTimeout: 60000 From b192cb65f0ae3ed4453dea095ce7c5f35198776c Mon Sep 17 00:00:00 2001 From: Paul Revello Date: Wed, 11 Jun 2025 12:51:28 -0400 Subject: [PATCH 02/28] Create cloudbuild.yaml --- .../manifests/cloudbuild.yaml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml new file mode 100644 index 00000000..5490aec8 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml @@ -0,0 +1,81 @@ +# --- Substitution Variables --- +substitutions: + _BACKSTAGE_APP_NAME: 'backstage-quickstart' + _LOCATION: 'us-central1' + _ARTIFACT_REGISTRY_REPO: 'backstage' + _IMAGE_NAME: 'backstage-quickstart' + +steps: + # Step 1: Scaffold the Backstage application into a subdirectory + - name: 'node:20' + id: 'Scaffold' + entrypoint: 'bash' + args: + - '-c' + # Use printf to provide the app name to the interactive prompt. + - 'printf "${_BACKSTAGE_APP_NAME}\n" | npx @backstage/create-app@latest --skip-install' + timeout: 1200s + + # Step 2: Run 'yarn install', overriding the default CI environment variable + - name: 'node:20' + id: 'Install Dependencies' + dir: '${_BACKSTAGE_APP_NAME}' + entrypoint: 'bash' + args: + - '-c' + # Set CI=false to allow yarn to create the lockfile in a CI environment + - 'CI=false yarn install' + waitFor: ['Scaffold'] + timeout: 1200s + + # Step 3: Build the application to create production artifacts + - name: 'node:20' + id: 'Build App' + dir: '${_BACKSTAGE_APP_NAME}' + entrypoint: 'yarn' + args: ['build:backend'] # 'yarn build' handles TypeScript compilation and everything else + # no need for yarn tsc + waitFor: ['Install Dependencies'] + timeout: 1200s + + # Nedd to paramatize a config file here so that it will rely on ENVARS + # Then create k8s secrets for those ENVARS + + # Step 4: Prepare the runtime config file + #- name: 'gcr.io/cloud-builders/gcloud' + # id: 'Prepare Config' + # dir: '${_BACKSTAGE_APP_NAME}' + # entrypoint: 'bash' + # args: + # - '-c' + # - | + # echo "Copying example production YAML" + # cp app-config.production.yaml.example app-config.production.yaml + # waitFor: ['Install Dependencies'] + + # Step 5: Build the Docker container + - name: 'gcr.io/cloud-builders/docker' + id: 'Create Container' + dir: '${_BACKSTAGE_APP_NAME}' + env: + - 'DOCKER_BUILDKIT=1' + args: + - 'build' + - '-t' + - '${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}' + - '--file=packages/backend/Dockerfile' + - '.' + waitFor: ['Build App'] #, 'Prepare Config'] + + # Step 6: Push the container image to Artifact Registry + - name: 'gcr.io/cloud-builders/docker' + id: 'Push' + dir: '${_BACKSTAGE_APP_NAME}' + args: + - 'push' + - '${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}' + waitFor: ['Create Container'] + +# --- Final Image Output --- +images: + - '${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}' From 71a9b4545983bed389678cc184ecf1577522dbd4 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Thu, 12 Jun 2025 17:22:49 +0000 Subject: [PATCH 03/28] Fixing the default AR Repo to align with the default value from Terraform. --- .../backstage/backstage-quickstart/manifests/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml index 5490aec8..5d2b2df3 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml @@ -2,7 +2,7 @@ substitutions: _BACKSTAGE_APP_NAME: 'backstage-quickstart' _LOCATION: 'us-central1' - _ARTIFACT_REGISTRY_REPO: 'backstage' + _ARTIFACT_REGISTRY_REPO: 'backstage-qs' _IMAGE_NAME: 'backstage-quickstart' steps: From a1d1bafdf93f85a4115feb771408a2dffaced877 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Tue, 17 Jun 2025 15:46:03 +0000 Subject: [PATCH 04/28] Adding resource to create an ssl cert. --- .../backstage/backstage-quickstart/endpoints.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/reference-architectures/backstage/backstage-quickstart/endpoints.tf b/reference-architectures/backstage/backstage-quickstart/endpoints.tf index d63bd8b9..9495d589 100644 --- a/reference-architectures/backstage/backstage-quickstart/endpoints.tf +++ b/reference-architectures/backstage/backstage-quickstart/endpoints.tf @@ -27,3 +27,11 @@ resource "google_endpoints_service" "backstageQsEndpoint" { project = var.environment_project_id service_name = local.backstageExternalUrl } + +resource "google_compute_managed_ssl_certificate" "backstageCert" { + name = "backstage-qs-cert" + + managed { + domains = [backstageExternalUrl] + } +} \ No newline at end of file From 6253c3ce6a69f787263760bde01025dcca408a15 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Tue, 17 Jun 2025 16:03:38 +0000 Subject: [PATCH 05/28] some fixes from testing. --- .../backstage/backstage-quickstart/db.tf | 20 +++++++++++++++++++ .../backstage-quickstart/endpoints.tf | 2 +- ...uction.yaml => app-config.production.yaml} | 0 3 files changed, 21 insertions(+), 1 deletion(-) rename reference-architectures/backstage/backstage-quickstart/manifests/{app-config-production.yaml => app-config.production.yaml} (100%) diff --git a/reference-architectures/backstage/backstage-quickstart/db.tf b/reference-architectures/backstage/backstage-quickstart/db.tf index 0267a2f0..8903935b 100644 --- a/reference-architectures/backstage/backstage-quickstart/db.tf +++ b/reference-architectures/backstage/backstage-quickstart/db.tf @@ -63,3 +63,23 @@ resource "null_resource" "sqlIamDelay" { "before" = "${google_sql_database_instance.instance.id}" } } + +data "template_file" "app_config_production" { + template = "${file("./manifests/app-config.production.yaml")}" + vars = { + POSTGRES_HOST = google_sql_database_instance.instance.dns_name + POSTGRES_PORT = 5432 + POSTGRES_DB = "backstage" + POSTGRES_USER = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") + } +} + +resource "null_resource" "local" { + # triggers { + # template = "${data.template_file.test.rendered}" + # } + + provisioner "local-exec" { + command = "echo \"${data.template_file.app_config_production.rendered}\" > ./manifests/app-config.production.yaml.rendered" + } +} diff --git a/reference-architectures/backstage/backstage-quickstart/endpoints.tf b/reference-architectures/backstage/backstage-quickstart/endpoints.tf index 9495d589..b440425f 100644 --- a/reference-architectures/backstage/backstage-quickstart/endpoints.tf +++ b/reference-architectures/backstage/backstage-quickstart/endpoints.tf @@ -32,6 +32,6 @@ resource "google_compute_managed_ssl_certificate" "backstageCert" { name = "backstage-qs-cert" managed { - domains = [backstageExternalUrl] + domains = [local.backstageExternalUrl] } } \ No newline at end of file diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/app-config-production.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/app-config.production.yaml similarity index 100% rename from reference-architectures/backstage/backstage-quickstart/manifests/app-config-production.yaml rename to reference-architectures/backstage/backstage-quickstart/manifests/app-config.production.yaml From fb30977ae1844f2648424f9166e89df9c1b62a36 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Wed, 18 Jun 2025 19:26:51 +0000 Subject: [PATCH 06/28] Organizing and adding in manifests. --- .../backstage/backstage-quickstart/db.tf | 31 +++++-------- .../backstage-quickstart/endpoints.tf | 4 +- .../backstage/backstage-quickstart/iap.tf | 45 +++++++++++++++++++ .../{ => cloudbuild}/cloudbuild.yaml | 35 +++++++-------- .../manifests/k8s/backstage.yaml | 43 ++++++++++++++++++ .../app-config.production.tftpl.yaml} | 12 ++--- .../backstage-qs-endpoint-spec-tftpl.yaml | 0 .../gateway-external-https.tftpl.yaml | 32 +++++++++++++ .../gcp-backend-policy-iap-service.tftpl.yaml | 31 +++++++++++++ .../templates/http-route-service.tftpl.yaml | 30 +++++++++++++ .../templates/oauth-secret.tftpl.yaml | 22 +++++++++ .../backstage/backstage-quickstart/vpc.tf | 20 +++++++++ 12 files changed, 259 insertions(+), 46 deletions(-) rename reference-architectures/backstage/backstage-quickstart/manifests/{ => cloudbuild}/cloudbuild.yaml (73%) create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml rename reference-architectures/backstage/backstage-quickstart/manifests/{app-config.production.yaml => templates/app-config.production.tftpl.yaml} (54%) rename reference-architectures/backstage/backstage-quickstart/{ => manifests/templates}/backstage-qs-endpoint-spec-tftpl.yaml (100%) create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/gateway-external-https.tftpl.yaml create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/gcp-backend-policy-iap-service.tftpl.yaml create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/oauth-secret.tftpl.yaml diff --git a/reference-architectures/backstage/backstage-quickstart/db.tf b/reference-architectures/backstage/backstage-quickstart/db.tf index 8903935b..164aa121 100644 --- a/reference-architectures/backstage/backstage-quickstart/db.tf +++ b/reference-architectures/backstage/backstage-quickstart/db.tf @@ -49,7 +49,7 @@ resource "google_sql_database" "database" { resource "google_sql_user" "iam_service_account_user" { # Note: for Postgres only, GCP requires omitting the ".gserviceaccount.com" suffix # from the service account email due to length limits on database usernames. - + name = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") instance = google_sql_database_instance.instance.name type = "CLOUD_IAM_SERVICE_ACCOUNT" @@ -64,22 +64,15 @@ resource "null_resource" "sqlIamDelay" { } } -data "template_file" "app_config_production" { - template = "${file("./manifests/app-config.production.yaml")}" - vars = { - POSTGRES_HOST = google_sql_database_instance.instance.dns_name - POSTGRES_PORT = 5432 - POSTGRES_DB = "backstage" - POSTGRES_USER = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") - } -} - -resource "null_resource" "local" { - # triggers { - # template = "${data.template_file.test.rendered}" - # } - - provisioner "local-exec" { - command = "echo \"${data.template_file.app_config_production.rendered}\" > ./manifests/app-config.production.yaml.rendered" - } +resource "local_file" "app_config_production_yaml" { + content = templatefile( + "${path.module}/manifests/templates/app-config.production.tftpl.yaml", + { + postgres_host = google_sql_database_instance.instance.dns_name + postgres_port = 5432 + postgres_db = "backstage" + postgres_user = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") + } + ) + filename = "${path.module}/manifests/cloudbuild/app-config.production.yaml" } diff --git a/reference-architectures/backstage/backstage-quickstart/endpoints.tf b/reference-architectures/backstage/backstage-quickstart/endpoints.tf index b440425f..9c010fb6 100644 --- a/reference-architectures/backstage/backstage-quickstart/endpoints.tf +++ b/reference-architectures/backstage/backstage-quickstart/endpoints.tf @@ -18,7 +18,7 @@ locals { resource "google_endpoints_service" "backstageQsEndpoint" { openapi_config = templatefile( - "${path.module}/backstage-qs-endpoint-spec-tftpl.yaml", + "${path.module}/manifests/templates/backstage-qs-endpoint-spec-tftpl.yaml", { endpoint = local.backstageExternalUrl, ip_address = google_compute_global_address.backstageQsEndpointAddress.address @@ -34,4 +34,4 @@ resource "google_compute_managed_ssl_certificate" "backstageCert" { managed { domains = [local.backstageExternalUrl] } -} \ No newline at end of file +} diff --git a/reference-architectures/backstage/backstage-quickstart/iap.tf b/reference-architectures/backstage/backstage-quickstart/iap.tf index 42abc65b..f973c420 100644 --- a/reference-architectures/backstage/backstage-quickstart/iap.tf +++ b/reference-architectures/backstage/backstage-quickstart/iap.tf @@ -28,3 +28,48 @@ resource "google_iap_client" "backstageIapClient" { display_name = var.backstage_iap_display_name brand = google_iap_brand.backstageIapBrand.name } + +resource "local_file" "route_https_yaml" { + content = templatefile( + "${path.module}/manifests/templates/http-route-service.tftpl.yaml", + { + gateway_name = local.gateway_name + hostname = local.backstageExternalUrl + http_route_name = "backstage-https" + namespace = "backstage" + service_name = "backstage" + service_port = 80 + } + ) + filename = "./manifests/k8s/http-route-service.yaml" +} + +############################################################################### +# IAP Policy +############################################################################### + +resource "local_file" "iap_secret_yaml" { + content = templatefile( + "${path.module}/manifests/templates/oauth-secret.tftpl.yaml", + { + name = "backstage-oauth" + namespace = "backstage" + secret = base64encode(google_iap_client.backstageIapClient.secret) + } + ) + filename = "./manifests/k8s/oauth-secret.yaml" +} + +resource "local_file" "policy_iap_backstage_yaml" { + content = templatefile( + "${path.module}/manifests/templates/gcp-backend-policy-iap-service.tftpl.yaml", + { + oauth_client_id = google_iap_client.backstageIapClient.client_id + oauth_client_secret_name = "backstage-oauth" + policy_name = "backstage" + service_name = "backstage" + namespace = "backstage" + } + ) + filename = "./manifests/k8s/gcp-backend-policy-iap-service.yaml" +} diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml similarity index 73% rename from reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml rename to reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml index 5d2b2df3..03619b3f 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml @@ -28,31 +28,28 @@ steps: waitFor: ['Scaffold'] timeout: 1200s - # Step 3: Build the application to create production artifacts + # Step 3: Prepare the runtime config file + - name: 'gcr.io/cloud-builders/gcloud' + id: 'Prepare Config' + dir: '${_BACKSTAGE_APP_NAME}' + entrypoint: 'bash' + args: + - '-c' + - | + echo "Copying rendered production YAML" + cp /workspace/app-config.production.yaml.rendered app-config.production.yaml + waitFor: ['Install Dependencies'] + + # Step 4: Build the application to create production artifacts - name: 'node:20' id: 'Build App' dir: '${_BACKSTAGE_APP_NAME}' entrypoint: 'yarn' - args: ['build:backend'] # 'yarn build' handles TypeScript compilation and everything else + args: ['build:backend', '--config', '../../app-config.yaml', '--config', '../../app-config.production.yaml'] # 'yarn build' handles TypeScript compilation and everything else # no need for yarn tsc - waitFor: ['Install Dependencies'] + waitFor: ['Prepare Config'] timeout: 1200s - # Nedd to paramatize a config file here so that it will rely on ENVARS - # Then create k8s secrets for those ENVARS - - # Step 4: Prepare the runtime config file - #- name: 'gcr.io/cloud-builders/gcloud' - # id: 'Prepare Config' - # dir: '${_BACKSTAGE_APP_NAME}' - # entrypoint: 'bash' - # args: - # - '-c' - # - | - # echo "Copying example production YAML" - # cp app-config.production.yaml.example app-config.production.yaml - # waitFor: ['Install Dependencies'] - # Step 5: Build the Docker container - name: 'gcr.io/cloud-builders/docker' id: 'Create Container' @@ -65,7 +62,7 @@ steps: - '${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}' - '--file=packages/backend/Dockerfile' - '.' - waitFor: ['Build App'] #, 'Prepare Config'] + waitFor: ['Build App', 'Prepare Config'] # Step 6: Push the container image to Artifact Registry - name: 'gcr.io/cloud-builders/docker' diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml new file mode 100644 index 00000000..c7764c79 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backstage + labels: + name: backstage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backstage + namespace: backstage +spec: + replicas: 1 + selector: + matchLabels: + app: backstage + template: + metadata: + labels: + app: backstage + spec: + containers: + - name: backstage + image: us-central1-docker.pkg.dev/backstage-460715/backstage-qs/backstage-quickstart:50f8851f-e409-4cd4-a0bb-bd6cc8801142 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 7007 +--- +apiVersion: v1 +kind: Service +metadata: + name: backstage + namespace: backstage +spec: + selector: + app: backstage + ports: + - name: http + port: 80 + targetPort: http + diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/app-config.production.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml similarity index 54% rename from reference-architectures/backstage/backstage-quickstart/manifests/app-config.production.yaml rename to reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml index 936bf0de..98b6dee6 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/app-config.production.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml @@ -1,12 +1,12 @@ database: client: pg connection: - host: ${POSTGRES_HOST} - port: ${POSTGRES_PORT} - database: ${POSTGRES_DB} - #user: backstage-qs-workload@backstage-453421.iam[this is just to show that the IAM account is trimmed when we authenticate to the cloudsql via IAM] - user: ${POSTGRES_USER} + host: ${postgres_host} + port: ${postgres_port} + database: ${postgres_db} + #user: backstage-qs-workload@backstage-453421.iam[this is just to show that the IAM account is trimmed when we authenticate to the cloudsql via IAM] + user: ${postgres_user} ssl: - rejectUnauthorized: false + rejectUnauthorized: false options: requestTimeout: 60000 diff --git a/reference-architectures/backstage/backstage-quickstart/backstage-qs-endpoint-spec-tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/backstage-qs-endpoint-spec-tftpl.yaml similarity index 100% rename from reference-architectures/backstage/backstage-quickstart/backstage-qs-endpoint-spec-tftpl.yaml rename to reference-architectures/backstage/backstage-quickstart/manifests/templates/backstage-qs-endpoint-spec-tftpl.yaml diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/gateway-external-https.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/gateway-external-https.tftpl.yaml new file mode 100644 index 00000000..d46a51e8 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/gateway-external-https.tftpl.yaml @@ -0,0 +1,32 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: ${gateway_name} + namespace: ${namespace} +spec: + addresses: + - type: NamedAddress + value: ${address_name} + gatewayClassName: gke-l7-global-external-managed + listeners: + - name: https + protocol: HTTPS + port: 443 + tls: + mode: Terminate + options: + networking.gke.io/pre-shared-certs: ${ssl_certificate_name} diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/gcp-backend-policy-iap-service.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/gcp-backend-policy-iap-service.tftpl.yaml new file mode 100644 index 00000000..2b13ca59 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/gcp-backend-policy-iap-service.tftpl.yaml @@ -0,0 +1,31 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: networking.gke.io/v1 +kind: GCPBackendPolicy +metadata: + name: ${policy_name} + namespace: ${namespace} +spec: + default: + iap: + clientID: ${oauth_client_id} + enabled: true + oauth2ClientSecret: + name: ${oauth_client_secret_name} + timeoutSec: 3600 + targetRef: + group: "" + kind: Service + name: ${service_name} diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml new file mode 100644 index 00000000..b5f86343 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: ${http_route_name} + namespace: ${namespace} +spec: + parentRefs: + - kind: Gateway + name: ${gateway_name} + hostnames: + - ${hostname} + rules: + - backendRefs: + - name: ${service_name} + port: ${service_port} + \ No newline at end of file diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/oauth-secret.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/oauth-secret.tftpl.yaml new file mode 100644 index 00000000..1322389e --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/oauth-secret.tftpl.yaml @@ -0,0 +1,22 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${name} + namespace: ${namespace} +type: Opaque +data: + secret: ${secret} diff --git a/reference-architectures/backstage/backstage-quickstart/vpc.tf b/reference-architectures/backstage/backstage-quickstart/vpc.tf index e7ec0f52..e5b999c9 100644 --- a/reference-architectures/backstage/backstage-quickstart/vpc.tf +++ b/reference-architectures/backstage/backstage-quickstart/vpc.tf @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +locals { + gateway_name = "external-https" +} resource "google_compute_network" "backstageHostingVpc" { project = var.environment_project_id @@ -108,3 +111,20 @@ resource "google_compute_global_address" "backstageQsEndpointAddress" { name = var.backstageqs_endpoint_address_name project = var.environment_project_id } + +resource "local_file" "gateway_external_https_yaml" { + depends_on = [ + google_compute_global_address.backstageQsEndpointAddress + ] + + content = templatefile( + "${path.module}/manifests/templates/gateway-external-https.tftpl.yaml", + { + address_name = google_compute_global_address.backstageQsEndpointAddress.name + gateway_name = local.gateway_name + namespace = "backstage" + ssl_certificate_name = google_compute_managed_ssl_certificate.backstageCert.name + } + ) + filename = "${path.module}/manifests/k8s/gateway-external-https.yaml" +} From 30f9f0d75062c8724404275ab417534202747ca8 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Wed, 18 Jun 2025 19:29:58 +0000 Subject: [PATCH 07/28] Cleaning up headers. --- .../manifests/cloudbuild/cloudbuild.yaml | 14 ++++++++++++++ .../manifests/k8s/backstage.yaml | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml index 03619b3f..b11fb8b5 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml @@ -1,3 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # --- Substitution Variables --- substitutions: _BACKSTAGE_APP_NAME: 'backstage-quickstart' diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml index c7764c79..d3fd1525 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml @@ -1,3 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- apiVersion: v1 kind: Namespace metadata: From 7416259a2e2093144708fdcdda853aa6b2c158b3 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Mon, 7 Jul 2025 21:48:36 +0000 Subject: [PATCH 08/28] back merging in changes from debugging. --- .../backstage/backstage-quickstart/db.tf | 5 +- .../backstage/backstage-quickstart/gke.tf | 2 - .../backstage/backstage-quickstart/iam.tf | 32 ++++++++++ .../backstage/backstage-quickstart/k8s.tf | 52 ++++++++++++++++ .../manifests/cloudbuild/cloudbuild.yaml | 31 +++++----- .../backstage-quickstart/manifests/k8s/.keep | 0 .../manifests/k8s/backstage.yaml | 57 ------------------ .../app-config.production.tftpl.yaml | 30 ++++++---- .../manifests/templates/deployment.tftpl.yaml | 60 +++++++++++++++++++ .../manifests/templates/namespace.tftpl.yaml | 20 +++++++ .../templates/service-account.tftpl.yaml | 21 +++++++ .../manifests/templates/service.tftpl.yaml | 26 ++++++++ .../backstage/backstage-quickstart/vpc.tf | 16 +++++ 13 files changed, 260 insertions(+), 92 deletions(-) create mode 100644 reference-architectures/backstage/backstage-quickstart/k8s.tf create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/k8s/.keep delete mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/namespace.tftpl.yaml create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/templates/service.tftpl.yaml diff --git a/reference-architectures/backstage/backstage-quickstart/db.tf b/reference-architectures/backstage/backstage-quickstart/db.tf index 164aa121..dae344aa 100644 --- a/reference-architectures/backstage/backstage-quickstart/db.tf +++ b/reference-architectures/backstage/backstage-quickstart/db.tf @@ -68,10 +68,7 @@ resource "local_file" "app_config_production_yaml" { content = templatefile( "${path.module}/manifests/templates/app-config.production.tftpl.yaml", { - postgres_host = google_sql_database_instance.instance.dns_name - postgres_port = 5432 - postgres_db = "backstage" - postgres_user = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") + endpoint_url = local.backstageExternalUrl } ) filename = "${path.module}/manifests/cloudbuild/app-config.production.yaml" diff --git a/reference-architectures/backstage/backstage-quickstart/gke.tf b/reference-architectures/backstage/backstage-quickstart/gke.tf index 3a80209e..0bac6cd8 100644 --- a/reference-architectures/backstage/backstage-quickstart/gke.tf +++ b/reference-architectures/backstage/backstage-quickstart/gke.tf @@ -28,7 +28,6 @@ resource "google_container_cluster" "hostingCluster" { } } - logging_config { enable_components = ["SYSTEM_COMPONENTS", "WORKLOADS"] } @@ -64,5 +63,4 @@ resource "google_container_cluster" "hostingCluster" { update = "30m" delete = "30m" } - } diff --git a/reference-architectures/backstage/backstage-quickstart/iam.tf b/reference-architectures/backstage/backstage-quickstart/iam.tf index d5b015f6..5b767da4 100644 --- a/reference-architectures/backstage/backstage-quickstart/iam.tf +++ b/reference-architectures/backstage/backstage-quickstart/iam.tf @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +### +# Hosting Service Account +### + resource "google_service_account" "hostingSa" { project = var.environment_project_id account_id = var.hosting_sa_id @@ -58,6 +62,10 @@ resource "time_sleep" "wait_30_seconds" { create_duration = "30s" } +### +# Workload Service Account +### + resource "google_service_account_iam_policy" "workloadIdentity" { depends_on = [google_container_cluster.hostingCluster, time_sleep.wait_30_seconds] service_account_id = google_service_account.workloadSa.name @@ -86,8 +94,32 @@ resource "google_project_iam_member" "cloudSqlBinding" { member = "serviceAccount:${google_service_account.workloadSa.email}" } +resource "google_project_iam_member" "cloudSqlClientBinding" { + project = var.environment_project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.workloadSa.email}" +} + +resource "google_project_iam_member" "cloudSqlInstanceUserBinding" { + project = var.environment_project_id + role = "roles/cloudsql.instanceUser" + member = "serviceAccount:${google_service_account.workloadSa.email}" +} + resource "google_project_iam_member" "cloudStorageBinding" { project = var.environment_project_id role = "roles/storage.objectUser" member = "serviceAccount:${google_service_account.workloadSa.email}" } + +resource "local_file" "route_https_yaml" { + content = templatefile( + "${path.module}/manifests/templates/service-account.tftpl.yaml", + { + service_account_name = "ksa-backstage" + namespace = "backstage" + gcp_service_account = google_service_account.workloadSa.email + } + ) + filename = "./manifests/k8s/service-account.yaml" +} diff --git a/reference-architectures/backstage/backstage-quickstart/k8s.tf b/reference-architectures/backstage/backstage-quickstart/k8s.tf new file mode 100644 index 00000000..019ae1e1 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/k8s.tf @@ -0,0 +1,52 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "local_file" "namespace_yaml" { + content = templatefile( + "${path.module}/manifests/templates/namespace.tftpl.yaml", + { + namespace = "backstage" + } + ) + filename = "./manifests/k8s/namespace.yaml" +} + +resource "local_file" "service_yaml" { + content = templatefile( + "${path.module}/manifests/templates/service.tftpl.yaml", + { + deployment_name = "backstage" + namespace = "backstage" + service_name = "backstage" + service_port = 80 + } + ) + filename = "./manifests/k8s/service.yaml" +} + +resource "local_file" "deployment_yaml" { + content = templatefile( + "${path.module}/manifests/templates/deployment.tftpl.yaml", + { + cloud_sql_name = google_sql_database_instance.instance.dns_name + deployment_name = "backstage" + namespace = "backstage" + postgres_port = 5432 + postgres_db = "backstage" + postgres_user = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") + service_account_name = "ksa-backstage" + } + ) + filename = "./manifests/k8s/deployment.yaml" +} diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml index b11fb8b5..a7f5ec88 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml @@ -1,17 +1,3 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # --- Substitution Variables --- substitutions: _BACKSTAGE_APP_NAME: 'backstage-quickstart' @@ -39,9 +25,21 @@ steps: - '-c' # Set CI=false to allow yarn to create the lockfile in a CI environment - 'CI=false yarn install' + - 'yarn --cwd packages/backend add pg' waitFor: ['Scaffold'] timeout: 1200s + - name: 'node:20' + id: 'Install pg backend' + dir: '${_BACKSTAGE_APP_NAME}' + entrypoint: 'bash' + args: + - '-c' + # Set CI=false to allow yarn to create the lockfile in a CI environment + - 'CI=false yarn --cwd packages/backend add pg' + waitFor: ['Install Dependencies'] + timeout: 1200s + # Step 3: Prepare the runtime config file - name: 'gcr.io/cloud-builders/gcloud' id: 'Prepare Config' @@ -50,9 +48,8 @@ steps: args: - '-c' - | - echo "Copying rendered production YAML" - cp /workspace/app-config.production.yaml.rendered app-config.production.yaml - waitFor: ['Install Dependencies'] + cp /workspace/app-config.production.yaml app-config.production.yaml + waitFor: ['Install pg backend'] # Step 4: Build the application to create production artifacts - name: 'node:20' diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/k8s/.keep b/reference-architectures/backstage/backstage-quickstart/manifests/k8s/.keep new file mode 100644 index 00000000..e69de29b diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml deleted file mode 100644 index d3fd1525..00000000 --- a/reference-architectures/backstage/backstage-quickstart/manifests/k8s/backstage.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -apiVersion: v1 -kind: Namespace -metadata: - name: backstage - labels: - name: backstage ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: backstage - namespace: backstage -spec: - replicas: 1 - selector: - matchLabels: - app: backstage - template: - metadata: - labels: - app: backstage - spec: - containers: - - name: backstage - image: us-central1-docker.pkg.dev/backstage-460715/backstage-qs/backstage-quickstart:50f8851f-e409-4cd4-a0bb-bd6cc8801142 - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 7007 ---- -apiVersion: v1 -kind: Service -metadata: - name: backstage - namespace: backstage -spec: - selector: - app: backstage - ports: - - name: http - port: 80 - targetPort: http - diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml index 98b6dee6..4fee52f9 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml @@ -1,12 +1,18 @@ -database: - client: pg - connection: - host: ${postgres_host} - port: ${postgres_port} - database: ${postgres_db} - #user: backstage-qs-workload@backstage-453421.iam[this is just to show that the IAM account is trimmed when we authenticate to the cloudsql via IAM] - user: ${postgres_user} - ssl: - rejectUnauthorized: false -options: -requestTimeout: 60000 +app: + baseUrl: https://${endpoint_url} + +backend: + baseUrl: https://${endpoint_url} + listen: + port: 7007 + database: + client: pg + connection: + host: ${POSTGRES_HOST} + port: ${POSTGRES_PORT} + database: ${POSTGRES_DB} + user: ${POSTGRES_USER} + # ssl: + # rejectUnauthorized: false + options: + requestTimeout: 60000 \ No newline at end of file diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml new file mode 100644 index 00000000..055eb15b --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml @@ -0,0 +1,60 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${deployment_name} + namespace: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app: ${deployment_name} + template: + metadata: + labels: + app: ${deployment_name} + spec: + serviceAccountName: ${service_account_name} + containers: + - name: cloud-sql-proxy-1 + image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.14.1 + args: + - "--structured-logs" + - "--port=5432" + - "--auto-iam-authn" + - "${cloud_sql_name}" + - "--psc" + securityContext: + runAsNonRoot: true + resources: + requests: + memory: "2Gi" + cpu: "1" + - name: backstage + image: CONTAINER_IMAGE + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 7007 + env: + - name: POSTGRES_USER + value: "${postgres_user}" + - name: POSTGRES_HOST + value: "127.0.0.1" + - name: POSTGRES_PORT + value: "${postgres_port}" + - name: POSTGRES_DB + value: "${postgres_db}" \ No newline at end of file diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/namespace.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/namespace.tftpl.yaml new file mode 100644 index 00000000..e1fc5e02 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/namespace.tftpl.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + name: ${namespace} diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml new file mode 100644 index 00000000..ed61aae8 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml @@ -0,0 +1,21 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + iam.gke.io/gcp-service-account: ${gcp_service_account} + name: ${service_account_name} + namespace: ${namespace} \ No newline at end of file diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/service.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/service.tftpl.yaml new file mode 100644 index 00000000..6a462680 --- /dev/null +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/service.tftpl.yaml @@ -0,0 +1,26 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: Service +metadata: + name: ${service_name} + namespace: ${namespace} +spec: + selector: + app: ${deployment_name} + ports: + - name: http + port: ${service_port} + targetPort: http diff --git a/reference-architectures/backstage/backstage-quickstart/vpc.tf b/reference-architectures/backstage/backstage-quickstart/vpc.tf index e5b999c9..72ac0e09 100644 --- a/reference-architectures/backstage/backstage-quickstart/vpc.tf +++ b/reference-architectures/backstage/backstage-quickstart/vpc.tf @@ -107,6 +107,22 @@ resource "google_compute_forwarding_rule" "cloudSqlPscForwardingRule" { target = google_sql_database_instance.instance.psc_service_attachment_link } +# Firewall rule required to support IAM logins to Cloud SQL +# https://cloud.google.com/sql/docs/postgres/iam-logins +resource "google_compute_firewall" "cloud_sql_auth" { + name = "cloudsql_auth" + network = google_compute_network.backstageHostingVpc.id + + allow { + protocol = "tcp" + ports = ["443", "3307"] + } + direction = "EGRESS" + destination_ranges = ["34.126.0.0/18"] + target_service_accounts = [google_service_account.workloadSa.email] +} + + resource "google_compute_global_address" "backstageQsEndpointAddress" { name = var.backstageqs_endpoint_address_name project = var.environment_project_id From f4234b1bfc76f9aad629b0396905232f54ab827c Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Tue, 8 Jul 2025 15:53:05 +0000 Subject: [PATCH 09/28] fixing issues found in testing up to terraform plan --- .../backstage/backstage-quickstart/README.md | 2 +- .../backstage/backstage-quickstart/iam.tf | 2 +- .../manifests/templates/app-config.production.tftpl.yaml | 8 ++++---- .../backstage/backstage-quickstart/vpc.tf | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md index c18c4919..ad3927b5 100644 --- a/reference-architectures/backstage/backstage-quickstart/README.md +++ b/reference-architectures/backstage/backstage-quickstart/README.md @@ -103,7 +103,7 @@ Management API are enabled. 1. Enable Service Usage API and Service Management API ```bash - gcloud services enable serviceusage.googleapis.com \ + gcloud services enable serviceusage.googleapis.com gcloud services enable servicemanagement.googleapis.com ``` diff --git a/reference-architectures/backstage/backstage-quickstart/iam.tf b/reference-architectures/backstage/backstage-quickstart/iam.tf index 5b767da4..6686268b 100644 --- a/reference-architectures/backstage/backstage-quickstart/iam.tf +++ b/reference-architectures/backstage/backstage-quickstart/iam.tf @@ -112,7 +112,7 @@ resource "google_project_iam_member" "cloudStorageBinding" { member = "serviceAccount:${google_service_account.workloadSa.email}" } -resource "local_file" "route_https_yaml" { +resource "local_file" "service_account_yaml" { content = templatefile( "${path.module}/manifests/templates/service-account.tftpl.yaml", { diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml index 4fee52f9..cad2431e 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml @@ -8,10 +8,10 @@ backend: database: client: pg connection: - host: ${POSTGRES_HOST} - port: ${POSTGRES_PORT} - database: ${POSTGRES_DB} - user: ${POSTGRES_USER} + host: $${POSTGRES_HOST} + port: $${POSTGRES_PORT} + database: $${POSTGRES_DB} + user: $${POSTGRES_USER} # ssl: # rejectUnauthorized: false options: diff --git a/reference-architectures/backstage/backstage-quickstart/vpc.tf b/reference-architectures/backstage/backstage-quickstart/vpc.tf index 72ac0e09..7846c11d 100644 --- a/reference-architectures/backstage/backstage-quickstart/vpc.tf +++ b/reference-architectures/backstage/backstage-quickstart/vpc.tf @@ -110,7 +110,7 @@ resource "google_compute_forwarding_rule" "cloudSqlPscForwardingRule" { # Firewall rule required to support IAM logins to Cloud SQL # https://cloud.google.com/sql/docs/postgres/iam-logins resource "google_compute_firewall" "cloud_sql_auth" { - name = "cloudsql_auth" + name = "cloudsql-auth" network = google_compute_network.backstageHostingVpc.id allow { From d2854571d5b402f3f22bad681cd569f41c179479 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Tue, 8 Jul 2025 20:30:19 +0000 Subject: [PATCH 10/28] Migrating over more fixes and documenation updates from testing. --- .../backstage/backstage-quickstart/README.md | 48 +++++++++++++++++++ .../backstage/backstage-quickstart/k8s.tf | 2 +- .../backstage-quickstart/variables.tf | 1 + 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md index ad3927b5..d3f7a60e 100644 --- a/reference-architectures/backstage/backstage-quickstart/README.md +++ b/reference-architectures/backstage/backstage-quickstart/README.md @@ -120,6 +120,54 @@ Management API are enabled. This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes. +3. Build the container image for Backstage + + ```bash + cd manifests/cloudbuild + gcloud builds submit . + ``` + + The output of that command will include a fully qualified image path + similar to: `us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc` + Using that value create a new environment variable. + + ```bash + export IMAGE_PATH="" + ``` + +4. Configure Cloud SQL postgres user for password authentication. + + ```bash + gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password + ``` + +5. Grant the backstage workload service account create database permissions. + + a. In the Cloud Console, navigate to `SQL` + + b. Select the database instance + + c. In the left menu select `Cloud SQL Studio` + + d. Choose the `postgres` database and login with the `postgres` user and password you created in step 4. + + e. Run the following sql commands, to grant create database permissions + + ```sql + ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB + ``` + +6. Deploy the Kubernetes manifests + + ```bash + cd ../k8s + sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml + gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-enpoint + kubectl apply -f . + ``` + +7. In a browser navigate to you backstage endpoint. The url will be similar to `https://qs.endpoints.[your_project_id].cloud.goog` + ## Cleanup 1. Destroy the resources using Terraform destroy diff --git a/reference-architectures/backstage/backstage-quickstart/k8s.tf b/reference-architectures/backstage/backstage-quickstart/k8s.tf index 019ae1e1..b862b5ad 100644 --- a/reference-architectures/backstage/backstage-quickstart/k8s.tf +++ b/reference-architectures/backstage/backstage-quickstart/k8s.tf @@ -39,7 +39,7 @@ resource "local_file" "deployment_yaml" { content = templatefile( "${path.module}/manifests/templates/deployment.tftpl.yaml", { - cloud_sql_name = google_sql_database_instance.instance.dns_name + cloud_sql_name = google_sql_database_instance.instance.name deployment_name = "backstage" namespace = "backstage" postgres_port = 5432 diff --git a/reference-architectures/backstage/backstage-quickstart/variables.tf b/reference-architectures/backstage/backstage-quickstart/variables.tf index 05205c34..8eb5ff7f 100644 --- a/reference-architectures/backstage/backstage-quickstart/variables.tf +++ b/reference-architectures/backstage/backstage-quickstart/variables.tf @@ -44,6 +44,7 @@ variable "backstage_hosting_project_services" { default = [ "cloudresourcemanager.googleapis.com", "artifactregistry.googleapis.com", + "cloudbuild.googleapis.com", "compute.googleapis.com", "container.googleapis.com", "autoscaling.googleapis.com", From b23fb4d47bee5c22a8eda9d113c4b2ba3bb53c92 Mon Sep 17 00:00:00 2001 From: Benjamin Good Date: Tue, 8 Jul 2025 20:34:05 +0000 Subject: [PATCH 11/28] Cleaning up linting errors. --- .../backstage/backstage-quickstart/README.md | 19 +-- .../backstage/backstage-quickstart/iam.tf | 6 +- .../backstage/backstage-quickstart/iap.tf | 6 +- .../backstage/backstage-quickstart/k8s.tf | 22 ++-- .../manifests/cloudbuild/cloudbuild.yaml | 110 +++++++++--------- .../app-config.production.tftpl.yaml | 2 +- .../manifests/templates/deployment.tftpl.yaml | 18 +-- .../templates/http-route-service.tftpl.yaml | 1 - .../templates/service-account.tftpl.yaml | 2 +- .../backstage/backstage-quickstart/vpc.tf | 14 +-- 10 files changed, 104 insertions(+), 96 deletions(-) diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md index d3f7a60e..c3e2c664 100644 --- a/reference-architectures/backstage/backstage-quickstart/README.md +++ b/reference-architectures/backstage/backstage-quickstart/README.md @@ -120,28 +120,29 @@ Management API are enabled. This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes. -3. Build the container image for Backstage +3. Build the container image for Backstage ```bash cd manifests/cloudbuild gcloud builds submit . ``` - The output of that command will include a fully qualified image path - similar to: `us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc` + The output of that command will include a fully qualified image path similar + to: + `us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc` Using that value create a new environment variable. ```bash export IMAGE_PATH="" ``` -4. Configure Cloud SQL postgres user for password authentication. +4. Configure Cloud SQL postgres user for password authentication. ```bash gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password ``` -5. Grant the backstage workload service account create database permissions. +5. Grant the backstage workload service account create database permissions. a. In the Cloud Console, navigate to `SQL` @@ -149,7 +150,8 @@ Management API are enabled. c. In the left menu select `Cloud SQL Studio` - d. Choose the `postgres` database and login with the `postgres` user and password you created in step 4. + d. Choose the `postgres` database and login with the `postgres` user and + password you created in step 4. e. Run the following sql commands, to grant create database permissions @@ -157,7 +159,7 @@ Management API are enabled. ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB ``` -6. Deploy the Kubernetes manifests +6. Deploy the Kubernetes manifests ```bash cd ../k8s @@ -166,7 +168,8 @@ Management API are enabled. kubectl apply -f . ``` -7. In a browser navigate to you backstage endpoint. The url will be similar to `https://qs.endpoints.[your_project_id].cloud.goog` +7. In a browser navigate to you backstage endpoint. The URL will be similar to + `https://qs.endpoints.[your_project_id].cloud.goog` ## Cleanup diff --git a/reference-architectures/backstage/backstage-quickstart/iam.tf b/reference-architectures/backstage/backstage-quickstart/iam.tf index 6686268b..ce2733ad 100644 --- a/reference-architectures/backstage/backstage-quickstart/iam.tf +++ b/reference-architectures/backstage/backstage-quickstart/iam.tf @@ -116,9 +116,9 @@ resource "local_file" "service_account_yaml" { content = templatefile( "${path.module}/manifests/templates/service-account.tftpl.yaml", { - service_account_name = "ksa-backstage" - namespace = "backstage" - gcp_service_account = google_service_account.workloadSa.email + service_account_name = "ksa-backstage" + namespace = "backstage" + gcp_service_account = google_service_account.workloadSa.email } ) filename = "./manifests/k8s/service-account.yaml" diff --git a/reference-architectures/backstage/backstage-quickstart/iap.tf b/reference-architectures/backstage/backstage-quickstart/iap.tf index f973c420..5f566700 100644 --- a/reference-architectures/backstage/backstage-quickstart/iap.tf +++ b/reference-architectures/backstage/backstage-quickstart/iap.tf @@ -52,9 +52,9 @@ resource "local_file" "iap_secret_yaml" { content = templatefile( "${path.module}/manifests/templates/oauth-secret.tftpl.yaml", { - name = "backstage-oauth" - namespace = "backstage" - secret = base64encode(google_iap_client.backstageIapClient.secret) + name = "backstage-oauth" + namespace = "backstage" + secret = base64encode(google_iap_client.backstageIapClient.secret) } ) filename = "./manifests/k8s/oauth-secret.yaml" diff --git a/reference-architectures/backstage/backstage-quickstart/k8s.tf b/reference-architectures/backstage/backstage-quickstart/k8s.tf index b862b5ad..f3cc3b93 100644 --- a/reference-architectures/backstage/backstage-quickstart/k8s.tf +++ b/reference-architectures/backstage/backstage-quickstart/k8s.tf @@ -26,10 +26,10 @@ resource "local_file" "service_yaml" { content = templatefile( "${path.module}/manifests/templates/service.tftpl.yaml", { - deployment_name = "backstage" - namespace = "backstage" - service_name = "backstage" - service_port = 80 + deployment_name = "backstage" + namespace = "backstage" + service_name = "backstage" + service_port = 80 } ) filename = "./manifests/k8s/service.yaml" @@ -39,13 +39,13 @@ resource "local_file" "deployment_yaml" { content = templatefile( "${path.module}/manifests/templates/deployment.tftpl.yaml", { - cloud_sql_name = google_sql_database_instance.instance.name - deployment_name = "backstage" - namespace = "backstage" - postgres_port = 5432 - postgres_db = "backstage" - postgres_user = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") - service_account_name = "ksa-backstage" + cloud_sql_name = google_sql_database_instance.instance.name + deployment_name = "backstage" + namespace = "backstage" + postgres_port = 5432 + postgres_db = "backstage" + postgres_user = trimsuffix(google_service_account.workloadSa.email, ".gserviceaccount.com") + service_account_name = "ksa-backstage" } ) filename = "./manifests/k8s/deployment.yaml" diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml index a7f5ec88..b86860c2 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml @@ -1,89 +1,95 @@ # --- Substitution Variables --- substitutions: - _BACKSTAGE_APP_NAME: 'backstage-quickstart' - _LOCATION: 'us-central1' - _ARTIFACT_REGISTRY_REPO: 'backstage-qs' - _IMAGE_NAME: 'backstage-quickstart' + _BACKSTAGE_APP_NAME: "backstage-quickstart" + _LOCATION: "us-central1" + _ARTIFACT_REGISTRY_REPO: "backstage-qs" + _IMAGE_NAME: "backstage-quickstart" steps: # Step 1: Scaffold the Backstage application into a subdirectory - - name: 'node:20' - id: 'Scaffold' - entrypoint: 'bash' + - name: "node:20" + id: "Scaffold" + entrypoint: "bash" args: - - '-c' + - "-c" # Use printf to provide the app name to the interactive prompt. - 'printf "${_BACKSTAGE_APP_NAME}\n" | npx @backstage/create-app@latest --skip-install' timeout: 1200s # Step 2: Run 'yarn install', overriding the default CI environment variable - - name: 'node:20' - id: 'Install Dependencies' - dir: '${_BACKSTAGE_APP_NAME}' - entrypoint: 'bash' + - name: "node:20" + id: "Install Dependencies" + dir: "${_BACKSTAGE_APP_NAME}" + entrypoint: "bash" args: - - '-c' + - "-c" # Set CI=false to allow yarn to create the lockfile in a CI environment - - 'CI=false yarn install' - - 'yarn --cwd packages/backend add pg' - waitFor: ['Scaffold'] + - "CI=false yarn install" + - "yarn --cwd packages/backend add pg" + waitFor: ["Scaffold"] timeout: 1200s - - name: 'node:20' - id: 'Install pg backend' - dir: '${_BACKSTAGE_APP_NAME}' - entrypoint: 'bash' + - name: "node:20" + id: "Install pg backend" + dir: "${_BACKSTAGE_APP_NAME}" + entrypoint: "bash" args: - - '-c' + - "-c" # Set CI=false to allow yarn to create the lockfile in a CI environment - - 'CI=false yarn --cwd packages/backend add pg' - waitFor: ['Install Dependencies'] + - "CI=false yarn --cwd packages/backend add pg" + waitFor: ["Install Dependencies"] timeout: 1200s # Step 3: Prepare the runtime config file - - name: 'gcr.io/cloud-builders/gcloud' - id: 'Prepare Config' - dir: '${_BACKSTAGE_APP_NAME}' - entrypoint: 'bash' + - name: "gcr.io/cloud-builders/gcloud" + id: "Prepare Config" + dir: "${_BACKSTAGE_APP_NAME}" + entrypoint: "bash" args: - - '-c' + - "-c" - | cp /workspace/app-config.production.yaml app-config.production.yaml - waitFor: ['Install pg backend'] + waitFor: ["Install pg backend"] # Step 4: Build the application to create production artifacts - - name: 'node:20' - id: 'Build App' - dir: '${_BACKSTAGE_APP_NAME}' - entrypoint: 'yarn' - args: ['build:backend', '--config', '../../app-config.yaml', '--config', '../../app-config.production.yaml'] # 'yarn build' handles TypeScript compilation and everything else + - name: "node:20" + id: "Build App" + dir: "${_BACKSTAGE_APP_NAME}" + entrypoint: "yarn" + args: [ + "build:backend", + "--config", + "../../app-config.yaml", + "--config", + "../../app-config.production.yaml", + ] # 'yarn build' handles TypeScript compilation and everything else # no need for yarn tsc - waitFor: ['Prepare Config'] + waitFor: ["Prepare Config"] timeout: 1200s # Step 5: Build the Docker container - - name: 'gcr.io/cloud-builders/docker' - id: 'Create Container' - dir: '${_BACKSTAGE_APP_NAME}' + - name: "gcr.io/cloud-builders/docker" + id: "Create Container" + dir: "${_BACKSTAGE_APP_NAME}" env: - - 'DOCKER_BUILDKIT=1' + - "DOCKER_BUILDKIT=1" args: - - 'build' - - '-t' - - '${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}' - - '--file=packages/backend/Dockerfile' - - '.' - waitFor: ['Build App', 'Prepare Config'] + - "build" + - "-t" + - "${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}" + - "--file=packages/backend/Dockerfile" + - "." + waitFor: ["Build App", "Prepare Config"] # Step 6: Push the container image to Artifact Registry - - name: 'gcr.io/cloud-builders/docker' - id: 'Push' - dir: '${_BACKSTAGE_APP_NAME}' + - name: "gcr.io/cloud-builders/docker" + id: "Push" + dir: "${_BACKSTAGE_APP_NAME}" args: - - 'push' - - '${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}' - waitFor: ['Create Container'] + - "push" + - "${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}" + waitFor: ["Create Container"] # --- Final Image Output --- images: - - '${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}' + - "${_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REGISTRY_REPO}/${_IMAGE_NAME}:${BUILD_ID}" diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml index cad2431e..776c50b1 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml @@ -15,4 +15,4 @@ backend: # ssl: # rejectUnauthorized: false options: - requestTimeout: 60000 \ No newline at end of file + requestTimeout: 60000 diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml index 055eb15b..172f244a 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml @@ -42,7 +42,7 @@ spec: resources: requests: memory: "2Gi" - cpu: "1" + cpu: "1" - name: backstage image: CONTAINER_IMAGE imagePullPolicy: IfNotPresent @@ -50,11 +50,11 @@ spec: - name: http containerPort: 7007 env: - - name: POSTGRES_USER - value: "${postgres_user}" - - name: POSTGRES_HOST - value: "127.0.0.1" - - name: POSTGRES_PORT - value: "${postgres_port}" - - name: POSTGRES_DB - value: "${postgres_db}" \ No newline at end of file + - name: POSTGRES_USER + value: "${postgres_user}" + - name: POSTGRES_HOST + value: "127.0.0.1" + - name: POSTGRES_PORT + value: "${postgres_port}" + - name: POSTGRES_DB + value: "${postgres_db}" diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml index b5f86343..9dd89b96 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/http-route-service.tftpl.yaml @@ -27,4 +27,3 @@ spec: - backendRefs: - name: ${service_name} port: ${service_port} - \ No newline at end of file diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml index ed61aae8..44a1e18e 100644 --- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml +++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/service-account.tftpl.yaml @@ -18,4 +18,4 @@ metadata: annotations: iam.gke.io/gcp-service-account: ${gcp_service_account} name: ${service_account_name} - namespace: ${namespace} \ No newline at end of file + namespace: ${namespace} diff --git a/reference-architectures/backstage/backstage-quickstart/vpc.tf b/reference-architectures/backstage/backstage-quickstart/vpc.tf index 7846c11d..aa960204 100644 --- a/reference-architectures/backstage/backstage-quickstart/vpc.tf +++ b/reference-architectures/backstage/backstage-quickstart/vpc.tf @@ -13,7 +13,7 @@ # limitations under the License. locals { - gateway_name = "external-https" + gateway_name = "external-https" } resource "google_compute_network" "backstageHostingVpc" { @@ -110,15 +110,15 @@ resource "google_compute_forwarding_rule" "cloudSqlPscForwardingRule" { # Firewall rule required to support IAM logins to Cloud SQL # https://cloud.google.com/sql/docs/postgres/iam-logins resource "google_compute_firewall" "cloud_sql_auth" { - name = "cloudsql-auth" + name = "cloudsql-auth" network = google_compute_network.backstageHostingVpc.id - + allow { - protocol = "tcp" - ports = ["443", "3307"] + protocol = "tcp" + ports = ["443", "3307"] } - direction = "EGRESS" - destination_ranges = ["34.126.0.0/18"] + direction = "EGRESS" + destination_ranges = ["34.126.0.0/18"] target_service_accounts = [google_service_account.workloadSa.email] } From 7a14133dc42dce47b168194990e6192c6b22743f Mon Sep 17 00:00:00 2001 From: bgood <1019754+bgood@users.noreply.github.com> Date: Tue, 8 Jul 2025 20:43:35 +0000 Subject: [PATCH 12/28] chore: update documentation site --- .../backstage/backstage-quickstart/index.html | 152 +++++++++++------- docs/search/search_index.json | 2 +- docs/sitemap.xml | 38 ++--- docs/sitemap.xml.gz | Bin 437 -> 437 bytes 4 files changed, 116 insertions(+), 76 deletions(-) diff --git a/docs/reference-architectures/backstage/backstage-quickstart/index.html b/docs/reference-architectures/backstage/backstage-quickstart/index.html index d683b52b..ac022eac 100644 --- a/docs/reference-architectures/backstage/backstage-quickstart/index.html +++ b/docs/reference-architectures/backstage/backstage-quickstart/index.html @@ -1864,7 +1864,7 @@

Deploy Backstage
gcloud services enable serviceusage.googleapis.com \
+
gcloud services enable serviceusage.googleapis.com
 gcloud services enable servicemanagement.googleapis.com
 

 
@@ -1879,45 +1879,85 @@ 
Deploy Backstage
cd manifests/cloudbuild
+gcloud builds submit .
+

+
The output of that command will include a fully qualified image path similar
+to:
+us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc
+Using that value create a new environment variable.

+
export IMAGE_PATH="<your_image_path>"
+

+
+
  • 
+
    Configure Cloud SQL postgres user for password authentication.
    
+
    gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password
+
    
+
    • 
+
  • 
+
    Grant the backstage workload service account create database permissions.
    
+
    a. In the Cloud Console, navigate to SQL
    
+
    b. Select the database instance
    
+
    c. In the left menu select Cloud SQL Studio
    
+
    d. Choose the postgres database and login with the postgres user and
+password you created in step 4.
    
+
    e. Run the following sql commands, to grant create database permissions
    
+
    ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
+
    
+
    • 
+
  • 
+
    Deploy the Kubernetes manifests
    
+
    cd ../k8s
+sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
+gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-enpoint
+kubectl apply -f .
+
    
+
    • 
+
  • 
+
    In a browser navigate to you backstage endpoint. The URL will be similar to
+    https://qs.endpoints.[your_project_id].cloud.goog
    
+
    • 
 
 
    Cleanup
    
 
      
 
    1. 
 
      Destroy the resources using Terraform destroy
      
-
      cd ${BACKSTAGE_QS_BASE_DIR} && \
-terraform init && \
-terraform destroy -auto-approve && \
-rm -rf .terraform .terraform.lock.hcl
+
      cd ${BACKSTAGE_QS_BASE_DIR} && \
+terraform init && \
+terraform destroy -auto-approve && \
+rm -rf .terraform .terraform.lock.hcl
 
      
 
      2. 
 
    2. 
 
      Delete the project
      
-
      gcloud projects delete ${PROJECT_ID}
+
      gcloud projects delete ${PROJECT_ID}
 
      
 
      3. 
 
    3. 
 
      Remove Terraform files and temporary files
      
-
      cd ${BACKSTAGE_QS_BASE_DIR} && \
-rm -rf \
-.terraform \
-.terraform.lock.hcl \
-initialize/.terraform \
-initialize/.terraform.lock.hcl \
-initialize/backend.tf.local \
-initialize/state
+
      cd ${BACKSTAGE_QS_BASE_DIR} && \
+rm -rf \
+.terraform \
+.terraform.lock.hcl \
+initialize/.terraform \
+initialize/.terraform.lock.hcl \
+initialize/backend.tf.local \
+initialize/state
 
      
 
      4. 
 
    4. 
 
      Reset the TF variables file
      
-
      cd ${BACKSTAGE_QS_BASE_DIR} && \
-cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
+
      cd ${BACKSTAGE_QS_BASE_DIR} && \
+cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
 
      
 
      5. 
 
    5. 
 
      Remove the environment variables
      
-
      sed \
--i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
-${HOME}/.bashrc
+
      sed \
+-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
+${HOME}/.bashrc
 
      
 
      6. 
 
    
@@ -1939,17 +1979,17 @@ 
    Creating a Terraform managed proje
 
      
 
    1. 
 
      Set the configuration variables
      
-
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
+
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
 
      
-
      environment_name  = "qs"
-iapUserDomain = ""
-iapSupportEmail = ""
-project = {
-  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
-  folder_id          = "############"
-  name               = "backstage"
-  org_id             = "############"
-}
+
      environment_name  = "qs"
+iapUserDomain = ""
+iapSupportEmail = ""
+project = {
+  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
+  folder_id          = "############"
+  name               = "backstage"
+  org_id             = "############"
+}
 
      
 
      Values required :
      
 
        
@@ -1970,25 +2010,25 @@ 
        Creating a Terraform managed proje
 
 
      • 
 
        Authorize gcloud
        
-
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
+
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
 
        
 
        • 
 
      • 
 
        Create a new project
        
-
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
-terraform init && \
-terraform plan -input=false -out=tfplan && \
-terraform apply -input=false tfplan && \
-rm tfplan && \
-terraform init -force-copy -migrate-state && \
-rm -rf state
+
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
+terraform init && \
+terraform plan -input=false -out=tfplan && \
+terraform apply -input=false tfplan && \
+rm tfplan && \
+terraform init -force-copy -migrate-state && \
+rm -rf state
 
        
 
        • 
 
      • 
 
        Set the project environment variables in Cloud Shell
        
-
        PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
-awk -F"=" '{print $2}' | xargs)
+
        PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
+awk -F"=" '{print $2}' | xargs)
 
        
 
        • 
 
    
@@ -1996,15 +2036,15 @@ 
    Cleaning up a Terraform managed
 
      
 
    1. 
 
      Destroy the project
      
-
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
-TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
-xargs) && \
-cp backend.tf.local backend.tf && \
-terraform init -force-copy -lock=false -migrate-state && \
-gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
-terraform init && \
-terraform destroy -auto-approve  && \
-rm -rf .terraform .terraform.lock.hcl state/
+
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
+TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
+xargs) && \
+cp backend.tf.local backend.tf && \
+terraform init -force-copy -lock=false -migrate-state && \
+gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
+terraform init && \
+terraform destroy -auto-approve  && \
+rm -rf .terraform .terraform.lock.hcl state/
 
      
 
      2. 
 
    
@@ -2012,13 +2052,13 @@ 
    Re-using an Existing Project
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-gcloud endpoints services undelete \
-${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
---quiet 2>/dev/null
+
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+gcloud endpoints services undelete \
+${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
+--quiet 2>/dev/null
 
    
 
 
diff --git a/docs/search/search_index.json b/docs/search/search_index.json
index 5e281caa..11878e5a 100644
--- a/docs/search/search_index.json
+++ b/docs/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable serviceusage.googleapis.com \\\ngcloud services enable servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable serviceusage.googleapis.com\ngcloud services enable servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      3.  
    3.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      4.  
    4.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      5.  
    5.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      6.  
    6.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-enpoint\nkubectl apply -f .\n
       
      7.  
    7.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      8.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index 7b80539e..179ff41f 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -2,78 +2,78 @@
 
     
          https://googlecloudplatform.github.io/platform-engineering/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/code-of-conduct/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/contributing/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/accelerating-migrations/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/automated-password-rotation/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/backstage-quickstart/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudRun/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/WebsiteDemo/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/github-runners-gke/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/cli/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/gcp-sandboxes/
-         2025-06-18
+         2025-07-08
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/sandbox-modules/
-         2025-06-18
+         2025-07-08
     
 
\ No newline at end of file
diff --git a/docs/sitemap.xml.gz b/docs/sitemap.xml.gz
index 8950a2417e021f071016eadcace5ffe3d2cef246..949ac1bc0bedb6406684df143beff53e3880b760 100644
GIT binary patch
delta 420
zcmV;V0bBmH1GNJOABzYGfM0Bp2OWRT3S-^4ODQddJq^mn9*g7Xe5Q`&Npf@h_H%N#
zvbQm|hkAaAqA23`7mDm`ua8sMJOi=ExNF|DU9*8`arDvcnlGR4)m3xdUu-i42HBM0
z$h+nsl0IlX&vVOOU^0nYB&XRP$_?wYQ+MtAroY&fwIn!M>)Hmi{gQ=C8i#-A0@#3g
zOu@(=X=`lm}lbUp%5A=6vEB=|L-VVQ93FT2aPs(Yups}gedodEO;!9Qjx
z#R7$UWigJ~$}3}wLVn2d+KNW72P%yQ%8)&jcCnHPSQ~300GS0?P2MdQMy`WB3r{ezF`x`0gu3y(ZbP8DXg92Yejc9saorQwp0{>$>DG4*Gu_T
O?0*1iv&cMJ4gdhb#L(^l

delta 420
zcmV;V0bBmH1GNJOABzYGfB{mG2OWRD7FN1(moiogeHxUFJr>8&`Ai+lljP?1?dRlf
zWp87&4|V+#MN!1>FBI9?Tpz};dIDmPaoe1?>t+Sf;^3p(Hs8P9s;lO@JKJOm46-W0
zk+;oWBz@9)nx>Y$z-SV;NKUiemmAh6r(U-oo9=8?)|}vIEo&3dc5@akX&ir|3t$80
zAq68lq_K5g_PK98>OU?8(fJ5Kg-mbsoZ#<#hGoQ|yI5a*Q0q%|epNzFz9WEsCU|9*
zQY=uoQx@Zpt-LX|DCGMrZ>?wqJD}2NpbXi4X%{P*fVHs}0+5-3)#%-9VdOg4v*2h1
z29*r+gfu8ZSp-zU-y8eLVjO=|M*D23pV==aNfi>7Bo70mfWIDg0jEl-zs#Q79PRO2
zT>@^F=buv|d@_OGowBkKtG=f
zZ(zhK6dYeulw&N1W91&}=o>b1=gb0zg)_%
OV)q+3C|PG(4gdh_rpS!|


From a9a87e634b696972c14b560c662cefbaefe18f10 Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Wed, 9 Jul 2025 13:39:49 +0000
Subject: [PATCH 13/28] Fixing a typo in the gke credentials command.

---
 .../backstage/backstage-quickstart/README.md                    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index c3e2c664..ffcb1cf6 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -164,7 +164,7 @@ Management API are enabled.
     ```bash
     cd ../k8s
     sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
-    gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-enpoint
+    gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
     kubectl apply -f .
     ```
 

From 2231a71af6b18260242d797cbd4fbc464e246dae Mon Sep 17 00:00:00 2001
From: bgood <1019754+bgood@users.noreply.github.com>
Date: Wed, 9 Jul 2025 13:40:38 +0000
Subject: [PATCH 14/28] chore: update documentation site

---
 .../backstage/backstage-quickstart/index.html |   2 +-
 docs/search/search_index.json                 |   2 +-
 docs/sitemap.xml                              |  38 +++++++++---------
 docs/sitemap.xml.gz                           | Bin 437 -> 437 bytes
 4 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/docs/reference-architectures/backstage/backstage-quickstart/index.html b/docs/reference-architectures/backstage/backstage-quickstart/index.html
index ac022eac..92e8ed76 100644
--- a/docs/reference-architectures/backstage/backstage-quickstart/index.html
+++ b/docs/reference-architectures/backstage/backstage-quickstart/index.html
@@ -1911,7 +1911,7 @@ 
    Deploy Backstage
    cd ../k8s
 sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
-gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-enpoint
+gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
 kubectl apply -f .
 
    
 
diff --git a/docs/search/search_index.json b/docs/search/search_index.json
index 11878e5a..7cc210ca 100644
--- a/docs/search/search_index.json
+++ b/docs/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable serviceusage.googleapis.com\ngcloud services enable servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      3.  
    3.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      4.  
    4.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      5.  
    5.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      6.  
    6.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-enpoint\nkubectl apply -f .\n
       
      7.  
    7.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      8.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable serviceusage.googleapis.com\ngcloud services enable servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      3.  
    3.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      4.  
    4.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      5.  
    5.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      6.  
    6.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      7.  
    7.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      8.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index 179ff41f..ce5fd946 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -2,78 +2,78 @@
 
     
          https://googlecloudplatform.github.io/platform-engineering/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/code-of-conduct/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/contributing/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/accelerating-migrations/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/automated-password-rotation/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/backstage-quickstart/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudRun/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/WebsiteDemo/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/github-runners-gke/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/cli/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/gcp-sandboxes/
-         2025-07-08
+         2025-07-09
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/sandbox-modules/
-         2025-07-08
+         2025-07-09
     
 
\ No newline at end of file
diff --git a/docs/sitemap.xml.gz b/docs/sitemap.xml.gz
index 949ac1bc0bedb6406684df143beff53e3880b760..e520ca614a3c1e6f0f84510ffb96cd07cd64aabb 100644
GIT binary patch
delta 73
zcmV-P0Ji_N1GNJOABzYG0I_Y62Ot6KktjkZ`{ug8*k%e0vMIrlcg;a0eb9QI=a#*|
fWD>VXPP09f8`fv1?%J!dksJXCGYB_CS`GjJVmcl-

delta 73
zcmV-P0Ji_N1GNJOABzYGfM0Bp2Ot66ktjkZRdd~6Y%>K0*_7bOyXGL0K4?A9bIV>}
fGKpIxr`aCL4ePU0ckTPJksJXCYO}~ZS`GjJAFCbc


From 4ab6ef535d5a44de2c3b90993a43e48a2b3362f3 Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Wed, 9 Jul 2025 13:50:21 +0000
Subject: [PATCH 15/28] Alphabetizing and adding the iam api.

---
 .../backstage/backstage-quickstart/variables.tf       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/variables.tf b/reference-architectures/backstage/backstage-quickstart/variables.tf
index 8eb5ff7f..6eb84966 100644
--- a/reference-architectures/backstage/backstage-quickstart/variables.tf
+++ b/reference-architectures/backstage/backstage-quickstart/variables.tf
@@ -42,20 +42,21 @@ variable "backstage_hosting_project_services" {
   description = "Service APIs to enable"
   type        = list(string)
   default = [
-    "cloudresourcemanager.googleapis.com",
     "artifactregistry.googleapis.com",
+    "autoscaling.googleapis.com",
     "cloudbuild.googleapis.com",
+    "cloudresourcemanager.googleapis.com",
     "compute.googleapis.com",
     "container.googleapis.com",
-    "autoscaling.googleapis.com",
     "containerfilesystem.googleapis.com",
+    "dns.googleapis.com",
+    "iam.googleapis.com",
+    "iap.googleapis.com",
     "logging.googleapis.com",
     "monitoring.googleapis.com",
-    "sqladmin.googleapis.com",
     "secretmanager.googleapis.com",
     "servicenetworking.googleapis.com",
-    "dns.googleapis.com",
-    "iap.googleapis.com"
+    "sqladmin.googleapis.com"
   ]
 }
 

From cbd50b446400f9b3e411e137792abb166549fff6 Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Wed, 9 Jul 2025 18:04:03 +0000
Subject: [PATCH 16/28] Fixes for the IAP brand Terraform resource deprecation.

---
 .../backstage/backstage-quickstart/README.md      | 15 +++++++++++++--
 .../backstage/backstage-quickstart/iap.tf         |  8 +++-----
 .../manifests/cloudbuild/cloudbuild.yaml          |  8 ++++----
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index ffcb1cf6..1065cf7e 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -103,8 +103,19 @@ Management API are enabled.
 1.  Enable Service Usage API and Service Management API
 
     ```bash
-    gcloud services enable serviceusage.googleapis.com
-    gcloud services enable servicemanagement.googleapis.com
+    gcloud services enable \
+      iap.googleapis.com \
+      serviceusage.googleapis.com \
+      servicemanagement.googleapis.com
+    ```
+
+2.  Create the Identity Aware Proxy brand
+
+    ```bash
+    gcloud iap oauth-brands create \
+      --application_title="IAP Secured Backstage" \
+      --project="${PROJECT_ID}" \
+      --support_email="${IAP_SUPPORT_EMAIL}$"
     ```
 
 2.  Create the resources
diff --git a/reference-architectures/backstage/backstage-quickstart/iap.tf b/reference-architectures/backstage/backstage-quickstart/iap.tf
index 5f566700..4b0613d0 100644
--- a/reference-architectures/backstage/backstage-quickstart/iap.tf
+++ b/reference-architectures/backstage/backstage-quickstart/iap.tf
@@ -12,10 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-resource "google_iap_brand" "backstageIapBrand" {
-  project           = var.environment_project_id
-  support_email     = var.iap_support_email
-  application_title = var.backstage_iap_application_title
+data "google_project" "backstageProject" { 
+  project_id = var.environment_project_id
 }
 
 resource "google_iap_web_iam_member" "backstageIapPolicy" {
@@ -26,7 +24,7 @@ resource "google_iap_web_iam_member" "backstageIapPolicy" {
 
 resource "google_iap_client" "backstageIapClient" {
   display_name = var.backstage_iap_display_name
-  brand        = google_iap_brand.backstageIapBrand.name
+  brand        = "projects/${data.google_project.backstageProject.number}/brand/${data.google_project.backstageProject.number}"
 }
 
 resource "local_file" "route_https_yaml" {
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
index b86860c2..b427919e 100644
--- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
@@ -25,7 +25,6 @@ steps:
       - "-c"
       # Set CI=false to allow yarn to create the lockfile in a CI environment
       - "CI=false yarn install"
-      - "yarn --cwd packages/backend add pg"
     waitFor: ["Scaffold"]
     timeout: 1200s
 
@@ -34,10 +33,11 @@ steps:
     dir: "${_BACKSTAGE_APP_NAME}"
     entrypoint: "bash"
     args:
-      - "-c"
       # Set CI=false to allow yarn to create the lockfile in a CI environment
-      - "CI=false yarn --cwd packages/backend add pg"
-    waitFor: ["Install Dependencies"]
+      - "-c"      
+      - |
+        CI=false yarn --cwd packages/backend add pg
+    waitFor: ["Scaffold"]
     timeout: 1200s
 
   # Step 3: Prepare the runtime config file

From de7a4b781a79fd87ef474e9a38f15b6da892c972 Mon Sep 17 00:00:00 2001
From: bgood <1019754+bgood@users.noreply.github.com>
Date: Wed, 9 Jul 2025 18:04:41 +0000
Subject: [PATCH 17/28] chore: update documentation site

---
 .../backstage/backstage-quickstart/index.html | 152 ++++++++++--------
 docs/search/search_index.json                 |   2 +-
 2 files changed, 82 insertions(+), 72 deletions(-)

diff --git a/docs/reference-architectures/backstage/backstage-quickstart/index.html b/docs/reference-architectures/backstage/backstage-quickstart/index.html
index 92e8ed76..f2d9c6c0 100644
--- a/docs/reference-architectures/backstage/backstage-quickstart/index.html
+++ b/docs/reference-architectures/backstage/backstage-quickstart/index.html
@@ -1864,36 +1864,46 @@ 
    Deploy Backstage
    gcloud services enable serviceusage.googleapis.com
-gcloud services enable servicemanagement.googleapis.com
+
    gcloud services enable \
+  iap.googleapis.com \
+  serviceusage.googleapis.com \
+  servicemanagement.googleapis.com
+
    
+
+
  • 
+
    Create the Identity Aware Proxy brand
    
+
    gcloud iap oauth-brands create \
+  --application_title="IAP Secured Backstage" \
+  --project="${PROJECT_ID}" \
+  --support_email="${IAP_SUPPORT_EMAIL}$"
 
    
 
    • 
 
  • 
 
    Create the resources
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-terraform init && \
-terraform plan -input=false -out=tfplan && \
-terraform apply -input=false tfplan && \
-rm tfplan
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+terraform init && \
+terraform plan -input=false -out=tfplan && \
+terraform apply -input=false tfplan && \
+rm tfplan
 
    
 
    This will take a while to create all of the required resources, figure
 somewhere between 15 and 20 minutes.
    
 
    • 
 
  • 
 
    Build the container image for Backstage
    
-
    cd manifests/cloudbuild
-gcloud builds submit .
+
    cd manifests/cloudbuild
+gcloud builds submit .
 
    
 
    The output of that command will include a fully qualified image path similar
 to:
 us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc
 Using that value create a new environment variable.
    
-
    export IMAGE_PATH="<your_image_path>"
+
    export IMAGE_PATH="<your_image_path>"
 
    
 
    • 
 
  • 
 
    Configure Cloud SQL postgres user for password authentication.
    
-
    gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password
+
    gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password
 
    
 
    • 
 
  • 
@@ -1904,15 +1914,15 @@ 
    Deploy Backstage
    ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
+
    ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
 
    
 
    • 
 
  • 
 
    Deploy the Kubernetes manifests
    
-
    cd ../k8s
-sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
-gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
-kubectl apply -f .
+
    cd ../k8s
+sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
+gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
+kubectl apply -f .
 
    
 
    • 
 
  • 
@@ -1924,40 +1934,40 @@ 
    Cleanup
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-terraform init && \
-terraform destroy -auto-approve && \
-rm -rf .terraform .terraform.lock.hcl
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+terraform init && \
+terraform destroy -auto-approve && \
+rm -rf .terraform .terraform.lock.hcl
 
    
 
    • 
 
  • 
 
    Delete the project
    
-
    gcloud projects delete ${PROJECT_ID}
+
    gcloud projects delete ${PROJECT_ID}
 
    
 
    • 
 
  • 
 
    Remove Terraform files and temporary files
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-rm -rf \
-.terraform \
-.terraform.lock.hcl \
-initialize/.terraform \
-initialize/.terraform.lock.hcl \
-initialize/backend.tf.local \
-initialize/state
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+rm -rf \
+.terraform \
+.terraform.lock.hcl \
+initialize/.terraform \
+initialize/.terraform.lock.hcl \
+initialize/backend.tf.local \
+initialize/state
 
    
 
    • 
 
  • 
 
    Reset the TF variables file
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
 
    
 
    • 
 
  • 
 
    Remove the environment variables
    
-
    sed \
--i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
-${HOME}/.bashrc
+
    sed \
+-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
+${HOME}/.bashrc
 
    
 
    • 
 
@@ -1979,17 +1989,17 @@ 
    Creating a Terraform managed proje
 
      
 
    1. 
 
      Set the configuration variables
      
-
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
+
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
 
      
-
      environment_name  = "qs"
-iapUserDomain = ""
-iapSupportEmail = ""
-project = {
-  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
-  folder_id          = "############"
-  name               = "backstage"
-  org_id             = "############"
-}
+
      environment_name  = "qs"
+iapUserDomain = ""
+iapSupportEmail = ""
+project = {
+  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
+  folder_id          = "############"
+  name               = "backstage"
+  org_id             = "############"
+}
 
      
 
      Values required :
      
 
        
@@ -2010,25 +2020,25 @@ 
        Creating a Terraform managed proje
 
 
      • 
 
        Authorize gcloud
        
-
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
+
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
 
        
 
        • 
 
      • 
 
        Create a new project
        
-
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
-terraform init && \
-terraform plan -input=false -out=tfplan && \
-terraform apply -input=false tfplan && \
-rm tfplan && \
-terraform init -force-copy -migrate-state && \
-rm -rf state
+
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
+terraform init && \
+terraform plan -input=false -out=tfplan && \
+terraform apply -input=false tfplan && \
+rm tfplan && \
+terraform init -force-copy -migrate-state && \
+rm -rf state
 
        
 
        • 
 
      • 
 
        Set the project environment variables in Cloud Shell
        
-
        PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
-awk -F"=" '{print $2}' | xargs)
+
        PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
+awk -F"=" '{print $2}' | xargs)
 
        
 
        • 
 
    
@@ -2036,15 +2046,15 @@ 
    Cleaning up a Terraform managed
 
      
 
    1. 
 
      Destroy the project
      
-
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
-TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
-xargs) && \
-cp backend.tf.local backend.tf && \
-terraform init -force-copy -lock=false -migrate-state && \
-gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
-terraform init && \
-terraform destroy -auto-approve  && \
-rm -rf .terraform .terraform.lock.hcl state/
+
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
+TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
+xargs) && \
+cp backend.tf.local backend.tf && \
+terraform init -force-copy -lock=false -migrate-state && \
+gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
+terraform init && \
+terraform destroy -auto-approve  && \
+rm -rf .terraform .terraform.lock.hcl state/
 
      
 
      2. 
 
    
@@ -2052,13 +2062,13 @@ 
    Re-using an Existing Project
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-gcloud endpoints services undelete \
-${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
---quiet 2>/dev/null
+
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+gcloud endpoints services undelete \
+${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
+--quiet 2>/dev/null
 
    
 
 
diff --git a/docs/search/search_index.json b/docs/search/search_index.json
index 7cc210ca..229ee195 100644
--- a/docs/search/search_index.json
+++ b/docs/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable serviceusage.googleapis.com\ngcloud services enable servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      3.  
    3.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      4.  
    4.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      5.  
    5.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      6.  
    6.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      7.  
    7.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      8.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable \\\n  iap.googleapis.com \\\n  serviceusage.googleapis.com \\\n  servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the Identity Aware Proxy brand
       
      gcloud iap oauth-brands create \\\n  --application_title=\"IAP Secured Backstage\" \\\n  --project=\"${PROJECT_ID}\" \\\n  --support_email=\"${IAP_SUPPORT_EMAIL}$\"\n
       
      3.  
    3.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      4.  
    4.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      5.  
    5.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      6.  
    6.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      7.  
    7.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      8.  
    8.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      9.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file

From 299035642f8e1ff8f525a62f0244833e47390b94 Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Wed, 9 Jul 2025 18:18:35 +0000
Subject: [PATCH 18/28] Fixing the list numbers.

---
 .../backstage/backstage-quickstart/README.md         | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index 1065cf7e..345221f8 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -118,7 +118,7 @@ Management API are enabled.
       --support_email="${IAP_SUPPORT_EMAIL}$"
     ```
 
-2.  Create the resources
+3.  Create the resources
 
     ```bash
     cd ${BACKSTAGE_QS_BASE_DIR} && \
@@ -131,7 +131,7 @@ Management API are enabled.
     This will take a while to create all of the required resources, figure
     somewhere between 15 and 20 minutes.
 
-3.  Build the container image for Backstage
+4.  Build the container image for Backstage
 
     ```bash
     cd manifests/cloudbuild
@@ -147,13 +147,13 @@ Management API are enabled.
     export IMAGE_PATH=""
     ```
 
-4.  Configure Cloud SQL postgres user for password authentication.
+5.  Configure Cloud SQL postgres user for password authentication.
 
     ```bash
     gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password
     ```
 
-5.  Grant the backstage workload service account create database permissions.
+6.  Grant the backstage workload service account create database permissions.
 
     a. In the Cloud Console, navigate to `SQL`
 
@@ -170,7 +170,7 @@ Management API are enabled.
     ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
     ```
 
-6.  Deploy the Kubernetes manifests
+7.  Deploy the Kubernetes manifests
 
     ```bash
     cd ../k8s
@@ -179,7 +179,7 @@ Management API are enabled.
     kubectl apply -f .
     ```
 
-7.  In a browser navigate to you backstage endpoint. The URL will be similar to
+8.  In a browser navigate to you backstage endpoint. The URL will be similar to
     `https://qs.endpoints.[your_project_id].cloud.goog`
 
 ## Cleanup

From 4007578ee93f954e124dd780868067d982dc201e Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Wed, 9 Jul 2025 18:22:28 +0000
Subject: [PATCH 19/28] .... and tf format.

---
 reference-architectures/backstage/backstage-quickstart/iap.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/iap.tf b/reference-architectures/backstage/backstage-quickstart/iap.tf
index 4b0613d0..8f3f3d1f 100644
--- a/reference-architectures/backstage/backstage-quickstart/iap.tf
+++ b/reference-architectures/backstage/backstage-quickstart/iap.tf
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-data "google_project" "backstageProject" { 
+data "google_project" "backstageProject" {
   project_id = var.environment_project_id
 }
 

From dd3d42bd7788fee2bb32c7f63276309fd98146fd Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Thu, 10 Jul 2025 15:04:15 +0000
Subject: [PATCH 20/28] Linter.

---
 .../backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
index b427919e..30dca53a 100644
--- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
@@ -34,7 +34,7 @@ steps:
     entrypoint: "bash"
     args:
       # Set CI=false to allow yarn to create the lockfile in a CI environment
-      - "-c"      
+      - "-c"
       - |
         CI=false yarn --cwd packages/backend add pg
     waitFor: ["Scaffold"]

From 5619e2a5205cddc57818b26daa683c94d9d8f7c6 Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Thu, 10 Jul 2025 20:35:46 +0000
Subject: [PATCH 21/28] First cut at the IAP auth provider.

---
 .../backstage/backstage-quickstart/README.md  |  19 ++-
 .../manifests/cloudbuild/App.tsx              | 132 ++++++++++++++++++
 .../manifests/cloudbuild/cloudbuild.yaml      |   9 +-
 .../manifests/cloudbuild/index.ts             | 102 ++++++++++++++
 .../app-config.production.tftpl.yaml          |  16 +++
 .../manifests/templates/deployment.tftpl.yaml |   2 +
 6 files changed, 275 insertions(+), 5 deletions(-)
 create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/App.tsx
 create mode 100644 reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index 345221f8..b6bd1102 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -147,6 +147,8 @@ Management API are enabled.
     export IMAGE_PATH=""
     ```
 
+    This will take approximately 10 minutes to build and push the image.
+
 5.  Configure Cloud SQL postgres user for password authentication.
 
     ```bash
@@ -170,16 +172,29 @@ Management API are enabled.
     ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
     ```
 
-7.  Deploy the Kubernetes manifests
+7.  Capture the IAP audience 
+
+    a. In the Cloud Console, navigate to `Security` > `Identity-Aware Proxy`
+
+    b. Choose `Get JWT audience code` from the three dot menu on the right side of your Backend Service.
+
+    c. The value will be in the format of: `/projects//global/backendServices/`. Using that value create a new environment variable.
+
+    ```bash
+    export IAP_AUDIENCE_VALUE=""
+    ```
+
+8.  Deploy the Kubernetes manifests
 
     ```bash
     cd ../k8s
     sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
+    sed -i "s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g" deployment.yaml
     gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
     kubectl apply -f .
     ```
 
-8.  In a browser navigate to you backstage endpoint. The URL will be similar to
+9.  In a browser navigate to you backstage endpoint. The URL will be similar to
     `https://qs.endpoints.[your_project_id].cloud.goog`
 
 ## Cleanup
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/App.tsx b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/App.tsx
new file mode 100644
index 00000000..304a8092
--- /dev/null
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/App.tsx
@@ -0,0 +1,132 @@
+import { Navigate, Route } from 'react-router-dom';
+import { apiDocsPlugin, ApiExplorerPage } from '@backstage/plugin-api-docs';
+import {
+  CatalogEntityPage,
+  CatalogIndexPage,
+  catalogPlugin,
+} from '@backstage/plugin-catalog';
+import {
+  CatalogImportPage,
+  catalogImportPlugin,
+} from '@backstage/plugin-catalog-import';
+import { ScaffolderPage, scaffolderPlugin } from '@backstage/plugin-scaffolder';
+import { orgPlugin } from '@backstage/plugin-org';
+import { SearchPage } from '@backstage/plugin-search';
+import {
+  TechDocsIndexPage,
+  techdocsPlugin,
+  TechDocsReaderPage,
+} from '@backstage/plugin-techdocs';
+import { TechDocsAddons } from '@backstage/plugin-techdocs-react';
+import { ReportIssue } from '@backstage/plugin-techdocs-module-addons-contrib';
+import { UserSettingsPage } from '@backstage/plugin-user-settings';
+import { apis } from './apis';
+import { entityPage } from './components/catalog/EntityPage';
+import { searchPage } from './components/search/SearchPage';
+import { Root } from './components/Root';
+
+import {
+  AlertDisplay,
+  OAuthRequestDialog,
+  SignInPage,
+  ProxiedSignInPage,
+} from '@backstage/core-components';
+import { createApp } from '@backstage/app-defaults';
+import { AppRouter, FlatRoutes } from '@backstage/core-app-api';
+import { CatalogGraphPage } from '@backstage/plugin-catalog-graph';
+import { RequirePermission } from '@backstage/plugin-permission-react';
+import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/alpha';
+
+import {
+  configApiRef,
+  googleAuthApiRef,
+  useApi,
+} from '@backstage/core-plugin-api';
+
+const app = createApp({
+  apis,
+  bindRoutes({ bind }) {
+    bind(catalogPlugin.externalRoutes, {
+      createComponent: scaffolderPlugin.routes.root,
+      viewTechDoc: techdocsPlugin.routes.docRoot,
+      createFromTemplate: scaffolderPlugin.routes.selectedTemplate,
+    });
+    bind(apiDocsPlugin.externalRoutes, {
+      registerApi: catalogImportPlugin.routes.importPage,
+    });
+    bind(scaffolderPlugin.externalRoutes, {
+      registerComponent: catalogImportPlugin.routes.importPage,
+      viewTechDoc: techdocsPlugin.routes.docRoot,
+    });
+    bind(orgPlugin.externalRoutes, {
+      catalogIndex: catalogPlugin.routes.catalogIndex,
+    });
+  },
+  components: {
+    SignInPage: props => {
+      const configApi = useApi(configApiRef);
+      if (configApi.getString('auth.environment') === 'development') {
+        return (
+          
+        );
+      }
+      // Just uncomment this line,nothing else
+      return ;
+    },
+  },
+});
+
+const routes = (
+  
+    } />
+    } />
+    }
+    >
+      {entityPage}
+    
+    } />
+    }
+    >
+      
+        
+      
+    
+    } />
+    } />
+    
+          
+        
+      }
+    />
+    }>
+      {searchPage}
+    
+    } />
+    } />
+  
+);
+
+export default app.createRoot(
+  <>
+    
+    
+    
+      {routes}
+    
+  ,
+);
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
index 30dca53a..de171a28 100644
--- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/cloudbuild.yaml
@@ -29,7 +29,7 @@ steps:
     timeout: 1200s
 
   - name: "node:20"
-    id: "Install pg backend"
+    id: "Install Plugins"
     dir: "${_BACKSTAGE_APP_NAME}"
     entrypoint: "bash"
     args:
@@ -37,7 +37,8 @@ steps:
       - "-c"
       - |
         CI=false yarn --cwd packages/backend add pg
-    waitFor: ["Scaffold"]
+        CI=false yarn --cwd packages/backend add @backstage/plugin-auth-backend-module-gcp-iap-provider
+    waitFor: ["Install Dependencies"]
     timeout: 1200s
 
   # Step 3: Prepare the runtime config file
@@ -49,7 +50,9 @@ steps:
       - "-c"
       - |
         cp /workspace/app-config.production.yaml app-config.production.yaml
-    waitFor: ["Install pg backend"]
+        cp /workspace/index.ts packages/backend/src/index.ts
+        cp /workspace/App.tsx packages/app/src/App.tsx
+    waitFor: ["Install Plugins"]
 
   # Step 4: Build the application to create production artifacts
   - name: "node:20"
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts
new file mode 100644
index 00000000..26313144
--- /dev/null
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts
@@ -0,0 +1,102 @@
+/*
+ * Hi!
+ *
+ * Note that this is an EXAMPLE Backstage backend. Please check the README.
+ *
+ * Happy hacking!
+ */
+
+import { createBackend } from '@backstage/backend-defaults';
+
+//Custom auth resolver
+import { createBackendModule } from '@backstage/backend-plugin-api';
+import { gcpIapAuthenticator } from '@backstage/plugin-auth-backend-module-gcp-iap-provider';
+import {
+  authProvidersExtensionPoint,
+  createProxyAuthProviderFactory,
+} from '@backstage/plugin-auth-node';
+
+import { stringifyEntityRef } from '@backstage/catalog-model';
+
+const backend = createBackend();
+
+backend.add(import('@backstage/plugin-app-backend'));
+backend.add(import('@backstage/plugin-proxy-backend'));
+backend.add(import('@backstage/plugin-scaffolder-backend'));
+backend.add(import('@backstage/plugin-scaffolder-backend-module-github'));
+backend.add(import('@backstage/plugin-techdocs-backend'));
+
+// auth plugin
+backend.add(import('@backstage/plugin-auth-backend'));
+// See https://backstage.io/docs/backend-system/building-backends/migrating#the-auth-plugin
+// backend.add(import('@backstage/plugin-auth-backend-module-guest-provider'));
+// See https://backstage.io/docs/auth/guest/provider
+
+// catalog plugin
+backend.add(import('@backstage/plugin-catalog-backend'));
+backend.add(
+  import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'),
+);
+
+// See https://backstage.io/docs/features/software-catalog/configuration#subscribing-to-catalog-errors
+backend.add(import('@backstage/plugin-catalog-backend-module-logs'));
+
+// permission plugin
+backend.add(import('@backstage/plugin-permission-backend'));
+// See https://backstage.io/docs/permissions/getting-started for how to create your own permission policy
+backend.add(
+  import('@backstage/plugin-permission-backend-module-allow-all-policy'),
+);
+
+// search plugin
+backend.add(import('@backstage/plugin-search-backend'));
+
+// search engine
+// See https://backstage.io/docs/features/search/search-engines
+backend.add(import('@backstage/plugin-search-backend-module-pg'));
+
+// search collators
+backend.add(import('@backstage/plugin-search-backend-module-catalog'));
+backend.add(import('@backstage/plugin-search-backend-module-techdocs'));
+
+// kubernetes
+backend.add(import('@backstage/plugin-kubernetes-backend'));
+
+//Add custom resolver for IAP
+const customAuthResolver = createBackendModule({
+  pluginId: 'auth',
+  moduleId: 'custom-auth-provider',
+  register(reg) {
+    reg.registerInit({
+      deps: { providers: authProvidersExtensionPoint },
+      async init({ providers }) {
+        providers.registerProvider({
+          providerId: 'gcpIap',
+          factory: createProxyAuthProviderFactory({
+            authenticator: gcpIapAuthenticator,
+            async signInResolver(info, ctx) {
+              console.log(info);
+              //const { profile: { email } } = info;
+              const { result: { iapToken : { email } } } = info;
+              console.log(email)
+              const userEntity = stringifyEntityRef({
+                kind: 'User',
+                //name: userId,
+                name: email,
+                namespace: 'default',
+              });
+              return ctx.issueToken({
+                claims: {
+                  sub: userEntity,
+                  ent: [userEntity],
+                },
+              });
+            },
+          }),
+        });
+      },
+    });
+  },
+});
+backend.add(customAuthResolver);
+backend.start();
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml
index 776c50b1..f1dbdfb6 100644
--- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml
@@ -16,3 +16,19 @@ backend:
       #   rejectUnauthorized: false
       options:
       requestTimeout: 60000
+
+auth:
+  environment: production
+  providers:
+    gcpIap:
+      audience: $${IAP_AUDIENCE}
+      #jwtHeader: 'x-goog-iap-jwt-assertion' # Optional: Only if you are using a custom header for the IAP JWT
+      # signIn:
+      #   resolvers:
+      #     # See https://backstage.io/docs/auth/google/gcp-iap-auth#resolvers for more resolvers
+      #     - resolver: emailMatchingUserEntityProfileEmail
+      #     - resolver: emailMatchingUserEntityAnnotation
+
+catalog:
+  rules:
+    - allow: [Component, System, API, Resource, Location, Template, User, Group]
\ No newline at end of file
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml
index 172f244a..77027daf 100644
--- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/deployment.tftpl.yaml
@@ -58,3 +58,5 @@ spec:
               value: "${postgres_port}"
             - name: POSTGRES_DB
               value: "${postgres_db}"
+            - name: IAP_AUDIENCE
+              value: IAP_AUDIENCE_VALUE

From 2959ff6d5f9b2fe2b8e396aba8879bbf4baee999 Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Thu, 10 Jul 2025 20:37:21 +0000
Subject: [PATCH 22/28] linter

---
 .../backstage/backstage-quickstart/README.md  |  9 ++++---
 .../manifests/cloudbuild/index.ts             | 24 +++++++++++--------
 .../app-config.production.tftpl.yaml          |  2 +-
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index b6bd1102..78858fc9 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -172,13 +172,16 @@ Management API are enabled.
     ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
     ```
 
-7.  Capture the IAP audience 
+7.  Capture the IAP audience
 
     a. In the Cloud Console, navigate to `Security` > `Identity-Aware Proxy`
 
-    b. Choose `Get JWT audience code` from the three dot menu on the right side of your Backend Service.
+    b. Choose `Get JWT audience code` from the three dot menu on the right side
+    of your Backend Service.
 
-    c. The value will be in the format of: `/projects//global/backendServices/`. Using that value create a new environment variable.
+    c. The value will be in the format of:
+    `/projects//global/backendServices/`. Using
+    that value create a new environment variable.
 
     ```bash
     export IAP_AUDIENCE_VALUE=""
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts
index 26313144..de1eed45 100644
--- a/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/cloudbuild/index.ts
@@ -6,17 +6,17 @@
  * Happy hacking!
  */
 
-import { createBackend } from '@backstage/backend-defaults';
+import {createBackend} from '@backstage/backend-defaults';
 
 //Custom auth resolver
-import { createBackendModule } from '@backstage/backend-plugin-api';
-import { gcpIapAuthenticator } from '@backstage/plugin-auth-backend-module-gcp-iap-provider';
+import {createBackendModule} from '@backstage/backend-plugin-api';
+import {gcpIapAuthenticator} from '@backstage/plugin-auth-backend-module-gcp-iap-provider';
 import {
   authProvidersExtensionPoint,
   createProxyAuthProviderFactory,
 } from '@backstage/plugin-auth-node';
 
-import { stringifyEntityRef } from '@backstage/catalog-model';
+import {stringifyEntityRef} from '@backstage/catalog-model';
 
 const backend = createBackend();
 
@@ -35,7 +35,7 @@ backend.add(import('@backstage/plugin-auth-backend'));
 // catalog plugin
 backend.add(import('@backstage/plugin-catalog-backend'));
 backend.add(
-  import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'),
+  import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model')
 );
 
 // See https://backstage.io/docs/features/software-catalog/configuration#subscribing-to-catalog-errors
@@ -45,7 +45,7 @@ backend.add(import('@backstage/plugin-catalog-backend-module-logs'));
 backend.add(import('@backstage/plugin-permission-backend'));
 // See https://backstage.io/docs/permissions/getting-started for how to create your own permission policy
 backend.add(
-  import('@backstage/plugin-permission-backend-module-allow-all-policy'),
+  import('@backstage/plugin-permission-backend-module-allow-all-policy')
 );
 
 // search plugin
@@ -68,8 +68,8 @@ const customAuthResolver = createBackendModule({
   moduleId: 'custom-auth-provider',
   register(reg) {
     reg.registerInit({
-      deps: { providers: authProvidersExtensionPoint },
-      async init({ providers }) {
+      deps: {providers: authProvidersExtensionPoint},
+      async init({providers}) {
         providers.registerProvider({
           providerId: 'gcpIap',
           factory: createProxyAuthProviderFactory({
@@ -77,8 +77,12 @@ const customAuthResolver = createBackendModule({
             async signInResolver(info, ctx) {
               console.log(info);
               //const { profile: { email } } = info;
-              const { result: { iapToken : { email } } } = info;
-              console.log(email)
+              const {
+                result: {
+                  iapToken: {email},
+                },
+              } = info;
+              console.log(email);
               const userEntity = stringifyEntityRef({
                 kind: 'User',
                 //name: userId,
diff --git a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml
index f1dbdfb6..e4625c1c 100644
--- a/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml
+++ b/reference-architectures/backstage/backstage-quickstart/manifests/templates/app-config.production.tftpl.yaml
@@ -31,4 +31,4 @@ auth:
 
 catalog:
   rules:
-    - allow: [Component, System, API, Resource, Location, Template, User, Group]
\ No newline at end of file
+    - allow: [Component, System, API, Resource, Location, Template, User, Group]

From 56beb375edd914ab5541c88d45cd06f2ed2e22ec Mon Sep 17 00:00:00 2001
From: bgood <1019754+bgood@users.noreply.github.com>
Date: Thu, 10 Jul 2025 20:37:45 +0000
Subject: [PATCH 23/28] chore: update documentation site

---
 .../backstage/backstage-quickstart/index.html | 131 ++++++++++--------
 docs/search/search_index.json                 |   2 +-
 docs/sitemap.xml                              |  38 ++---
 docs/sitemap.xml.gz                           | Bin 437 -> 437 bytes
 4 files changed, 92 insertions(+), 79 deletions(-)

diff --git a/docs/reference-architectures/backstage/backstage-quickstart/index.html b/docs/reference-architectures/backstage/backstage-quickstart/index.html
index f2d9c6c0..91361cba 100644
--- a/docs/reference-architectures/backstage/backstage-quickstart/index.html
+++ b/docs/reference-architectures/backstage/backstage-quickstart/index.html
@@ -1900,6 +1900,7 @@ 
    Deploy Backstage
    export IMAGE_PATH="<your_image_path>"
 
    
+
    This will take approximately 10 minutes to build and push the image.
    
 
 
  • 
 
    Configure Cloud SQL postgres user for password authentication.
    
@@ -1918,11 +1919,23 @@ 
    Deploy Backstage
    export IAP_AUDIENCE_VALUE="<your_iap_audience_value>"
+
    
+
    • 
+
  • 
 
    Deploy the Kubernetes manifests
    
-
    cd ../k8s
-sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
-gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
-kubectl apply -f .
+
    cd ../k8s
+sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
+sed -i "s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g" deployment.yaml
+gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
+kubectl apply -f .
 
    
 
    • 
 
  • 
@@ -1934,40 +1947,40 @@ 
    Cleanup
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-terraform init && \
-terraform destroy -auto-approve && \
-rm -rf .terraform .terraform.lock.hcl
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+terraform init && \
+terraform destroy -auto-approve && \
+rm -rf .terraform .terraform.lock.hcl
 
    
 
    • 
 
  • 
 
    Delete the project
    
-
    gcloud projects delete ${PROJECT_ID}
+
    gcloud projects delete ${PROJECT_ID}
 
    
 
    • 
 
  • 
 
    Remove Terraform files and temporary files
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-rm -rf \
-.terraform \
-.terraform.lock.hcl \
-initialize/.terraform \
-initialize/.terraform.lock.hcl \
-initialize/backend.tf.local \
-initialize/state
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+rm -rf \
+.terraform \
+.terraform.lock.hcl \
+initialize/.terraform \
+initialize/.terraform.lock.hcl \
+initialize/backend.tf.local \
+initialize/state
 
    
 
    • 
 
  • 
 
    Reset the TF variables file
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
 
    
 
    • 
 
  • 
 
    Remove the environment variables
    
-
    sed \
--i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
-${HOME}/.bashrc
+
    sed \
+-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
+${HOME}/.bashrc
 
    
 
    • 
 
@@ -1989,17 +2002,17 @@ 
    Creating a Terraform managed proje
 
      
 
    1. 
 
      Set the configuration variables
      
-
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
+
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
 
      
-
      environment_name  = "qs"
-iapUserDomain = ""
-iapSupportEmail = ""
-project = {
-  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
-  folder_id          = "############"
-  name               = "backstage"
-  org_id             = "############"
-}
+
      environment_name  = "qs"
+iapUserDomain = ""
+iapSupportEmail = ""
+project = {
+  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
+  folder_id          = "############"
+  name               = "backstage"
+  org_id             = "############"
+}
 
      
 
      Values required :
      
 
        
@@ -2020,25 +2033,25 @@ 
        Creating a Terraform managed proje
 
 
      • 
 
        Authorize gcloud
        
-
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
+
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
 
        
 
        • 
 
      • 
 
        Create a new project
        
-
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
-terraform init && \
-terraform plan -input=false -out=tfplan && \
-terraform apply -input=false tfplan && \
-rm tfplan && \
-terraform init -force-copy -migrate-state && \
-rm -rf state
+
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
+terraform init && \
+terraform plan -input=false -out=tfplan && \
+terraform apply -input=false tfplan && \
+rm tfplan && \
+terraform init -force-copy -migrate-state && \
+rm -rf state
 
        
 
        • 
 
      • 
 
        Set the project environment variables in Cloud Shell
        
-
        PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
-awk -F"=" '{print $2}' | xargs)
+
        PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
+awk -F"=" '{print $2}' | xargs)
 
        
 
        • 
 
    
@@ -2046,15 +2059,15 @@ 
    Cleaning up a Terraform managed
 
      
 
    1. 
 
      Destroy the project
      
-
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
-TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
-xargs) && \
-cp backend.tf.local backend.tf && \
-terraform init -force-copy -lock=false -migrate-state && \
-gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
-terraform init && \
-terraform destroy -auto-approve  && \
-rm -rf .terraform .terraform.lock.hcl state/
+
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
+TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
+xargs) && \
+cp backend.tf.local backend.tf && \
+terraform init -force-copy -lock=false -migrate-state && \
+gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
+terraform init && \
+terraform destroy -auto-approve  && \
+rm -rf .terraform .terraform.lock.hcl state/
 
      
 
      2. 
 
    
@@ -2062,13 +2075,13 @@ 
    Re-using an Existing Project
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-gcloud endpoints services undelete \
-${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
---quiet 2>/dev/null
+
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+gcloud endpoints services undelete \
+${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
+--quiet 2>/dev/null
 
    
 
 
diff --git a/docs/search/search_index.json b/docs/search/search_index.json
index 229ee195..a9f2c19b 100644
--- a/docs/search/search_index.json
+++ b/docs/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable \\\n  iap.googleapis.com \\\n  serviceusage.googleapis.com \\\n  servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the Identity Aware Proxy brand
       
      gcloud iap oauth-brands create \\\n  --application_title=\"IAP Secured Backstage\" \\\n  --project=\"${PROJECT_ID}\" \\\n  --support_email=\"${IAP_SUPPORT_EMAIL}$\"\n
       
      3.  
    3.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      4.  
    4.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      5.  
    5.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      6.  
    6.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      7.  
    7.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      8.  
    8.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      9.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable \\\n  iap.googleapis.com \\\n  serviceusage.googleapis.com \\\n  servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the Identity Aware Proxy brand
       
      gcloud iap oauth-brands create \\\n  --application_title=\"IAP Secured Backstage\" \\\n  --project=\"${PROJECT_ID}\" \\\n  --support_email=\"${IAP_SUPPORT_EMAIL}$\"\n
       
      3.  
    3.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      4.  
    4.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      This will take approximately 10 minutes to build and push the image.
       
      5.  
    5.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      6.  
    6.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      7.  
    7.  
      Capture the IAP audience
       
      a. In the Cloud Console, navigate to Security > Identity-Aware Proxy
       
      b. Choose Get JWT audience code from the three dot menu on the right side of your Backend Service.
       
      c. The value will be in the format of: /projects/<your_project_number>/global/backendServices/<numeric_id>. Using that value create a new environment variable.
       
      export IAP_AUDIENCE_VALUE=\"<your_iap_audience_value>\"\n
       
      8.  
    8.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\nsed -i \"s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      9.  
    9.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      10.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index ce5fd946..ef7a7c2e 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -2,78 +2,78 @@
 
     
          https://googlecloudplatform.github.io/platform-engineering/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/code-of-conduct/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/contributing/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/accelerating-migrations/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/automated-password-rotation/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/backstage-quickstart/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudRun/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/WebsiteDemo/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/github-runners-gke/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/cli/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/gcp-sandboxes/
-         2025-07-09
+         2025-07-10
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/sandbox-modules/
-         2025-07-09
+         2025-07-10
     
 
\ No newline at end of file
diff --git a/docs/sitemap.xml.gz b/docs/sitemap.xml.gz
index e520ca614a3c1e6f0f84510ffb96cd07cd64aabb..1da70047378b4fe4e9cfb0069ffe166874566191 100644
GIT binary patch
literal 437
zcmV;m0ZRTKiwFpS0&i#n|8r?{Wo=<_E_iKh0Ns~OYuqpphVS_mi|?|WKHD5m8tGCTff3eLJ7-Um|Bk!7nNP4gJJkKqAfypFpk(_3GC^xLnPTjTdn*L%_
z){@|4t!o?5_DdEnX&j;pU<2kc1tWW;sdZisd1yWApDqQ_`3OLTOmFp);MaVHWx}z)
z>aMO;_eNcICFJZo0q7Tkf6P*f1q%1dVjQ!TSH>2F{E+3f6^&pIR2mJGA$ut8VkHx>
zHr7G_G7GSpyjv`cTnBp=oQ%Mzl3|{aMnx#AfGYS0W1m=zgUV=MEcGq>)g-Ax!kXl9
zgcR`m<38YADfR8*xzEv_zSTA0ZhihCCBkPD_`g&3F-jRzww~$qC*nROsA=&2UK&A}
z%MWE|F$^_JddRU-{I4)9r@}j!unGmI*A(R#%i&nLr#kwEO&kY20#`;0M>D0cc8;$V
f-PxpSt^e6l1ty2Tp)Hmi{gQ=C8i(it*noLV!N?wIYMqxu9$Jt3r%OR}J_1l7(_6hH_%)wlnQ-hc
zyUVw#d#Adq5_0yP0Q3vNKV~V#0)=~JF^<{FD`Sg7e#r9Libk*pDvbupkUf-kv62Z`
z8*3o|nFUx)-YphJu7f=bPDWr<$uQ4Iqau`5Ko$Ihu}>_wmUXfyv=-=+{g6RqTHNGYB_CS`GjJEneF4


From 9622b81f31bba93d28a33b9fcf80e9f41c611c60 Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Fri, 11 Jul 2025 17:01:29 +0000
Subject: [PATCH 24/28] Fixes from testing and handling of the IAP Client
 resource deprecation.

---
 .../backstage/backstage-quickstart/README.md  | 63 ++++++++++++++-----
 .../backstage-qs.auto.tfvars                  |  3 +-
 .../backstage/backstage-quickstart/iap.tf     | 17 +++--
 .../backstage/backstage-quickstart/k8s.tf     |  2 +-
 .../backstage-quickstart/variables.tf         | 14 +++--
 5 files changed, 67 insertions(+), 32 deletions(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index 78858fc9..952aa388 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -86,15 +86,6 @@ In this section you prepare your project for deployment.
     gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}
     ```
 
-3.  Set the configuration variables
-
-    ```bash
-    sed -i "s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g" ${BACKSTAGE_QS_BASE_DIR}/backend.tf
-    sed -i "s/YOUR_PROJECT_ID/${PROJECT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-    sed -i "s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-    sed -i "s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-    ```
-
 ## Deploy Backstage
 
 Before running Terraform, make sure that the Service Usage API and Service
@@ -104,18 +95,50 @@ Management API are enabled.
 
     ```bash
     gcloud services enable \
+      cloudresourcemanager.googleapis.com \
       iap.googleapis.com \
       serviceusage.googleapis.com \
       servicemanagement.googleapis.com
     ```
 
-2.  Create the Identity Aware Proxy brand
+2.  Setup the Identity Aware Proxy brand
 
     ```bash
     gcloud iap oauth-brands create \
       --application_title="IAP Secured Backstage" \
       --project="${PROJECT_ID}" \
-      --support_email="${IAP_SUPPORT_EMAIL}$"
+      --support_email="${IAP_SUPPORT_EMAIL}"
+    ```
+
+    Capture the brand name in an environment variable, it will be in the format of: `projects/[your_project_number]/brands/[your_project_number]`.
+
+    ```bash
+    export IAP_BRAND=
+    ```
+
+3.  Using the brand name create the IAP client. 
+
+    ```bash
+    gcloud iap oauth-clients create \
+      ${IAP_BRAND} \
+      --display_name="IAP Secured Backstage"
+    ```    
+
+    Capture the client_id and client_secret in environment variables. For the client_id we only need the last value of the string, it will be in the format of: `549085115274-ksi3n9n41tp1vif79dda5ofauk0ebes9.apps.googleusercontent.com`
+
+    ```bash
+    export IAP_CLIENT_ID=""
+    export IAP_SECRET=""
+    ```
+
+4.  Set the configuration variables
+
+    ```bash
+    sed -i "s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g" ${BACKSTAGE_QS_BASE_DIR}/backend.tf
+    sed -i "s/YOUR_PROJECT_ID/${PROJECT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+    sed -i "s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+    sed -i "s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+    sed -i "s/YOUR_IAP_SECRET/${IAP_SECRET}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
     ```
 
 3.  Create the resources
@@ -172,10 +195,21 @@ Management API are enabled.
     ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
     ```
 
-7.  Capture the IAP audience
+7.  Perform an initial deployment of Kubernetes resources.
+
+     ```bash
+    cd ../k8s
+    sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
+    gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
+    kubectl apply -f .
+    ```
+
+7.  Capture the IAP audience, the Backend Service may take a few minutes to appear.
 
     a. In the Cloud Console, navigate to `Security` > `Identity-Aware Proxy`
 
+    b. Verify the IAP option is set to enabled. If not enable it now.
+
     b. Choose `Get JWT audience code` from the three dot menu on the right side
     of your Backend Service.
 
@@ -187,13 +221,10 @@ Management API are enabled.
     export IAP_AUDIENCE_VALUE=""
     ```
 
-8.  Deploy the Kubernetes manifests
+8.  Redeploy the Kubernetes manifests with the IAP audience
 
     ```bash
-    cd ../k8s
-    sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
     sed -i "s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g" deployment.yaml
-    gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
     kubectl apply -f .
     ```
 
diff --git a/reference-architectures/backstage/backstage-quickstart/backstage-qs.auto.tfvars b/reference-architectures/backstage/backstage-quickstart/backstage-qs.auto.tfvars
index a4d6e8b0..f9608ced 100644
--- a/reference-architectures/backstage/backstage-quickstart/backstage-qs.auto.tfvars
+++ b/reference-architectures/backstage/backstage-quickstart/backstage-qs.auto.tfvars
@@ -1,4 +1,5 @@
 environment_name       = "qs"
 iap_user_domain        = "YOUR_IAP_USER_DOMAIN"
-iap_support_email      = "YOUR_IAP_SUPPORT_EMAIL"
+iap_client_id          = "YOUR_IAP_CLIENT_ID"
+iap_client_secret      = "YOUR_IAP_SECRET"
 environment_project_id = "YOUR_PROJECT_ID"
diff --git a/reference-architectures/backstage/backstage-quickstart/iap.tf b/reference-architectures/backstage/backstage-quickstart/iap.tf
index 8f3f3d1f..f7932ca5 100644
--- a/reference-architectures/backstage/backstage-quickstart/iap.tf
+++ b/reference-architectures/backstage/backstage-quickstart/iap.tf
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-data "google_project" "backstageProject" {
-  project_id = var.environment_project_id
-}
+# data "external" "iap_client" {
+#   program = ["bash", "gcloud", "iap", "oauth-clients", "list", var.iap_brand, "--format", "json"]
+# }
 
 resource "google_iap_web_iam_member" "backstageIapPolicy" {
   project = var.environment_project_id
@@ -22,11 +22,6 @@ resource "google_iap_web_iam_member" "backstageIapPolicy" {
   member  = "domain:${var.iap_user_domain}"
 }
 
-resource "google_iap_client" "backstageIapClient" {
-  display_name = var.backstage_iap_display_name
-  brand        = "projects/${data.google_project.backstageProject.number}/brand/${data.google_project.backstageProject.number}"
-}
-
 resource "local_file" "route_https_yaml" {
   content = templatefile(
     "${path.module}/manifests/templates/http-route-service.tftpl.yaml",
@@ -52,7 +47,8 @@ resource "local_file" "iap_secret_yaml" {
     {
       name      = "backstage-oauth"
       namespace = "backstage"
-      secret    = base64encode(google_iap_client.backstageIapClient.secret)
+      secret    = base64encode(var.iap_client_secret)
+      //secret    = base64encode(jsondecode(data.external.iap_client.result)[0].secret)
     }
   )
   filename = "./manifests/k8s/oauth-secret.yaml"
@@ -62,7 +58,8 @@ resource "local_file" "policy_iap_backstage_yaml" {
   content = templatefile(
     "${path.module}/manifests/templates/gcp-backend-policy-iap-service.tftpl.yaml",
     {
-      oauth_client_id          = google_iap_client.backstageIapClient.client_id
+      //oauth_client_id          = split("/", jsondecode(data.external.iap_client.result)[0].name)[4]
+      oauth_client_id          = var.iap_client_id
       oauth_client_secret_name = "backstage-oauth"
       policy_name              = "backstage"
       service_name             = "backstage"
diff --git a/reference-architectures/backstage/backstage-quickstart/k8s.tf b/reference-architectures/backstage/backstage-quickstart/k8s.tf
index f3cc3b93..d584e26f 100644
--- a/reference-architectures/backstage/backstage-quickstart/k8s.tf
+++ b/reference-architectures/backstage/backstage-quickstart/k8s.tf
@@ -39,7 +39,7 @@ resource "local_file" "deployment_yaml" {
   content = templatefile(
     "${path.module}/manifests/templates/deployment.tftpl.yaml",
     {
-      cloud_sql_name       = google_sql_database_instance.instance.name
+      cloud_sql_name       = google_sql_database_instance.instance.connection_name
       deployment_name      = "backstage"
       namespace            = "backstage"
       postgres_port        = 5432
diff --git a/reference-architectures/backstage/backstage-quickstart/variables.tf b/reference-architectures/backstage/backstage-quickstart/variables.tf
index 6eb84966..3e3a83b7 100644
--- a/reference-architectures/backstage/backstage-quickstart/variables.tf
+++ b/reference-architectures/backstage/backstage-quickstart/variables.tf
@@ -18,14 +18,20 @@ variable "environment_name" {
   default     = "qs"
 }
 
-variable "iap_support_email" {
-  description = "The email address for the IAP support contact"
+variable "iap_user_domain" {
+  description = "The base domain name for the GCP org users accessing Backstage through IAP"
   type        = string
 }
 
-variable "iap_user_domain" {
-  description = "The base domain name for the GCP org users accessing Backstage through IAP"
+variable "iap_client_id" {
+  description = "The id of the IAP client"
+  type        = string
+}
+
+variable "iap_client_secret" {
+  description = "OAuth secret for the IAP client"
   type        = string
+  sensitive   = true
 }
 
 variable "environment_project_id" {

From 02c82e0e40219a161ae1e28f781f5c5206d9b6ff Mon Sep 17 00:00:00 2001
From: Benjamin Good 
Date: Fri, 11 Jul 2025 17:03:50 +0000
Subject: [PATCH 25/28] Linter run.

---
 .../backstage/backstage-quickstart/README.md  | 31 +++++++++++--------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index 952aa388..5e2959df 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -110,21 +110,25 @@ Management API are enabled.
       --support_email="${IAP_SUPPORT_EMAIL}"
     ```
 
-    Capture the brand name in an environment variable, it will be in the format of: `projects/[your_project_number]/brands/[your_project_number]`.
+    Capture the brand name in an environment variable, it will be in the format
+    of: `projects/[your_project_number]/brands/[your_project_number]`.
 
     ```bash
     export IAP_BRAND=
     ```
 
-3.  Using the brand name create the IAP client. 
+3.  Using the brand name create the IAP client.
 
     ```bash
     gcloud iap oauth-clients create \
       ${IAP_BRAND} \
       --display_name="IAP Secured Backstage"
-    ```    
+    ```
 
-    Capture the client_id and client_secret in environment variables. For the client_id we only need the last value of the string, it will be in the format of: `549085115274-ksi3n9n41tp1vif79dda5ofauk0ebes9.apps.googleusercontent.com`
+    Capture the client_id and client_secret in environment variables. For the
+    client_id we only need the last value of the string, it will be in the
+    format of:
+    `549085115274-ksi3n9n41tp1vif79dda5ofauk0ebes9.apps.googleusercontent.com`
 
     ```bash
     export IAP_CLIENT_ID=""
@@ -141,7 +145,7 @@ Management API are enabled.
     sed -i "s/YOUR_IAP_SECRET/${IAP_SECRET}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
     ```
 
-3.  Create the resources
+5.  Create the resources
 
     ```bash
     cd ${BACKSTAGE_QS_BASE_DIR} && \
@@ -154,7 +158,7 @@ Management API are enabled.
     This will take a while to create all of the required resources, figure
     somewhere between 15 and 20 minutes.
 
-4.  Build the container image for Backstage
+6.  Build the container image for Backstage
 
     ```bash
     cd manifests/cloudbuild
@@ -172,13 +176,13 @@ Management API are enabled.
 
     This will take approximately 10 minutes to build and push the image.
 
-5.  Configure Cloud SQL postgres user for password authentication.
+7.  Configure Cloud SQL postgres user for password authentication.
 
     ```bash
     gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password
     ```
 
-6.  Grant the backstage workload service account create database permissions.
+8.  Grant the backstage workload service account create database permissions.
 
     a. In the Cloud Console, navigate to `SQL`
 
@@ -195,16 +199,17 @@ Management API are enabled.
     ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
     ```
 
-7.  Perform an initial deployment of Kubernetes resources.
+9.  Perform an initial deployment of Kubernetes resources.
 
-     ```bash
+    ```bash
     cd ../k8s
     sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
     gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
     kubectl apply -f .
     ```
 
-7.  Capture the IAP audience, the Backend Service may take a few minutes to appear.
+10. Capture the IAP audience, the Backend Service may take a few minutes to
+    appear.
 
     a. In the Cloud Console, navigate to `Security` > `Identity-Aware Proxy`
 
@@ -221,14 +226,14 @@ Management API are enabled.
     export IAP_AUDIENCE_VALUE=""
     ```
 
-8.  Redeploy the Kubernetes manifests with the IAP audience
+11. Redeploy the Kubernetes manifests with the IAP audience
 
     ```bash
     sed -i "s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g" deployment.yaml
     kubectl apply -f .
     ```
 
-9.  In a browser navigate to you backstage endpoint. The URL will be similar to
+12. In a browser navigate to you backstage endpoint. The URL will be similar to
     `https://qs.endpoints.[your_project_id].cloud.goog`
 
 ## Cleanup

From 6db1d114d660363e31ecf56a7370b6a5d6c3ebbf Mon Sep 17 00:00:00 2001
From: bgood <1019754+bgood@users.noreply.github.com>
Date: Fri, 11 Jul 2025 17:04:19 +0000
Subject: [PATCH 26/28] chore: update documentation site

---
 .../backstage/backstage-quickstart/index.html | 207 ++++++++++--------
 docs/search/search_index.json                 |   2 +-
 docs/sitemap.xml                              |  38 ++--
 docs/sitemap.xml.gz                           | Bin 437 -> 437 bytes
 4 files changed, 137 insertions(+), 110 deletions(-)

diff --git a/docs/reference-architectures/backstage/backstage-quickstart/index.html b/docs/reference-architectures/backstage/backstage-quickstart/index.html
index 91361cba..a76b79b1 100644
--- a/docs/reference-architectures/backstage/backstage-quickstart/index.html
+++ b/docs/reference-architectures/backstage/backstage-quickstart/index.html
@@ -1849,14 +1849,6 @@ 
    Project Configuration
    gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}
 
    
 
-
  • 
-
    Set the configuration variables
    
-
    sed -i "s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g" ${BACKSTAGE_QS_BASE_DIR}/backend.tf
-sed -i "s/YOUR_PROJECT_ID/${PROJECT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-sed -i "s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-sed -i "s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-
    
-
    • 
 
 
    Deploy Backstage
    
 
    Before running Terraform, make sure that the Service Usage API and Service
@@ -1864,47 +1856,75 @@ 
    Deploy Backstage
    gcloud services enable \
-  iap.googleapis.com \
-  serviceusage.googleapis.com \
-  servicemanagement.googleapis.com
+
    gcloud services enable \
+  cloudresourcemanager.googleapis.com \
+  iap.googleapis.com \
+  serviceusage.googleapis.com \
+  servicemanagement.googleapis.com
+
    
+
+
  • 
+
    Setup the Identity Aware Proxy brand
    
+
    gcloud iap oauth-brands create \
+  --application_title="IAP Secured Backstage" \
+  --project="${PROJECT_ID}" \
+  --support_email="${IAP_SUPPORT_EMAIL}"
+
    
+
    Capture the brand name in an environment variable, it will be in the format
+of: projects/[your_project_number]/brands/[your_project_number].
    
+
    export IAP_BRAND=<your_brand_name>
 
    
 
    • 
 
  • 
-
    Create the Identity Aware Proxy brand
    
-
    gcloud iap oauth-brands create \
-  --application_title="IAP Secured Backstage" \
-  --project="${PROJECT_ID}" \
-  --support_email="${IAP_SUPPORT_EMAIL}$"
+
    Using the brand name create the IAP client.
    
+
    gcloud iap oauth-clients create \
+  ${IAP_BRAND} \
+  --display_name="IAP Secured Backstage"
+
    
+
    Capture the client_id and client_secret in environment variables. For the
+client_id we only need the last value of the string, it will be in the
+format of:
+549085115274-ksi3n9n41tp1vif79dda5ofauk0ebes9.apps.googleusercontent.com
    
+
    export IAP_CLIENT_ID="<your_client_id>"
+export IAP_SECRET="<your_iap_secret>"
+
    
+
    • 
+
  • 
+
    Set the configuration variables
    
+
    sed -i "s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g" ${BACKSTAGE_QS_BASE_DIR}/backend.tf
+sed -i "s/YOUR_PROJECT_ID/${PROJECT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+sed -i "s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+sed -i "s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+sed -i "s/YOUR_IAP_SECRET/${IAP_SECRET}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
 
    
 
    • 
 
  • 
 
    Create the resources
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-terraform init && \
-terraform plan -input=false -out=tfplan && \
-terraform apply -input=false tfplan && \
-rm tfplan
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+terraform init && \
+terraform plan -input=false -out=tfplan && \
+terraform apply -input=false tfplan && \
+rm tfplan
 
    
 
    This will take a while to create all of the required resources, figure
 somewhere between 15 and 20 minutes.
    
 
    • 
 
  • 
 
    Build the container image for Backstage
    
-
    cd manifests/cloudbuild
-gcloud builds submit .
+
    cd manifests/cloudbuild
+gcloud builds submit .
 
    
 
    The output of that command will include a fully qualified image path similar
 to:
 us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc
 Using that value create a new environment variable.
    
-
    export IMAGE_PATH="<your_image_path>"
+
    export IMAGE_PATH="<your_image_path>"
 
    
 
    This will take approximately 10 minutes to build and push the image.
    
 
    • 
 
  • 
 
    Configure Cloud SQL postgres user for password authentication.
    
-
    gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password
+
    gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password
 
    
 
    • 
 
  • 
@@ -1915,27 +1935,34 @@ 
    Deploy Backstage
    ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
+
    ALTER USER "backstage-qs-workload@[your_project_id].iam" CREATEDB
+
    
+
    • 
+
  • 
+
    Perform an initial deployment of Kubernetes resources.
    
+
    cd ../k8s
+sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
+gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
+kubectl apply -f .
 
    
 
    • 
 
  • 
-
    Capture the IAP audience
    
+
    Capture the IAP audience, the Backend Service may take a few minutes to
+    appear.
    
 
    a. In the Cloud Console, navigate to Security > Identity-Aware Proxy
    
+
    b. Verify the IAP option is set to enabled. If not enable it now.
    
 
    b. Choose Get JWT audience code from the three dot menu on the right side
 of your Backend Service.
    
 
    c. The value will be in the format of:
 /projects/<your_project_number>/global/backendServices/<numeric_id>. Using
 that value create a new environment variable.
    
-
    export IAP_AUDIENCE_VALUE="<your_iap_audience_value>"
+
    export IAP_AUDIENCE_VALUE="<your_iap_audience_value>"
 
    
 
    • 
 
  • 
-
    Deploy the Kubernetes manifests
    
-
    cd ../k8s
-sed -i "s%CONTAINER_IMAGE%${IMAGE_PATH}%g" deployment.yaml
-sed -i "s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g" deployment.yaml
-gcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint
-kubectl apply -f .
+
    Redeploy the Kubernetes manifests with the IAP audience
    
+
    sed -i "s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g" deployment.yaml
+kubectl apply -f .
 
    
 
    • 
 
  • 
@@ -1947,40 +1974,40 @@ 
    Cleanup
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-terraform init && \
-terraform destroy -auto-approve && \
-rm -rf .terraform .terraform.lock.hcl
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+terraform init && \
+terraform destroy -auto-approve && \
+rm -rf .terraform .terraform.lock.hcl
 
    
 
    • 
 
  • 
 
    Delete the project
    
-
    gcloud projects delete ${PROJECT_ID}
+
    gcloud projects delete ${PROJECT_ID}
 
    
 
    • 
 
  • 
 
    Remove Terraform files and temporary files
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-rm -rf \
-.terraform \
-.terraform.lock.hcl \
-initialize/.terraform \
-initialize/.terraform.lock.hcl \
-initialize/backend.tf.local \
-initialize/state
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+rm -rf \
+.terraform \
+.terraform.lock.hcl \
+initialize/.terraform \
+initialize/.terraform.lock.hcl \
+initialize/backend.tf.local \
+initialize/state
 
    
 
    • 
 
  • 
 
    Reset the TF variables file
    
-
    cd ${BACKSTAGE_QS_BASE_DIR} && \
-cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
+
    cd ${BACKSTAGE_QS_BASE_DIR} && \
+cp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars
 
    
 
    • 
 
  • 
 
    Remove the environment variables
    
-
    sed \
--i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
-${HOME}/.bashrc
+
    sed \
+-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \
+${HOME}/.bashrc
 
    
 
    • 
 
@@ -2002,17 +2029,17 @@ 
    Creating a Terraform managed proje
 
      
 
    1. 
 
      Set the configuration variables
      
-
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
+
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars
 
      
-
      environment_name  = "qs"
-iapUserDomain = ""
-iapSupportEmail = ""
-project = {
-  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
-  folder_id          = "############"
-  name               = "backstage"
-  org_id             = "############"
-}
+
      environment_name  = "qs"
+iapUserDomain = ""
+iapSupportEmail = ""
+project = {
+  billing_account_id = "XXXXXX-XXXXXX-XXXXXX"
+  folder_id          = "############"
+  name               = "backstage"
+  org_id             = "############"
+}
 
      
 
      Values required :
      
 
        
@@ -2033,25 +2060,25 @@ 
        Creating a Terraform managed proje
 
 
      • 
 
        Authorize gcloud
        
-
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
+
        gcloud auth login --activate --no-launch-browser --quiet --update-adc
 
        
 
        • 
 
      • 
 
        Create a new project
        
-
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
-terraform init && \
-terraform plan -input=false -out=tfplan && \
-terraform apply -input=false tfplan && \
-rm tfplan && \
-terraform init -force-copy -migrate-state && \
-rm -rf state
+
        cd ${BACKSTAGE_QS_BASE_DIR}/initialize
+terraform init && \
+terraform plan -input=false -out=tfplan && \
+terraform apply -input=false tfplan && \
+rm tfplan && \
+terraform init -force-copy -migrate-state && \
+rm -rf state
 
        
 
        • 
 
      • 
 
        Set the project environment variables in Cloud Shell
        
-
        PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
-awk -F"=" '{print $2}' | xargs)
+
        PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |
+awk -F"=" '{print $2}' | xargs)
 
        
 
        • 
 
    
@@ -2059,15 +2086,15 @@ 
    Cleaning up a Terraform managed
 
      
 
    1. 
 
      Destroy the project
      
-
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
-TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
-xargs) && \
-cp backend.tf.local backend.tf && \
-terraform init -force-copy -lock=false -migrate-state && \
-gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
-terraform init && \
-terraform destroy -auto-approve  && \
-rm -rf .terraform .terraform.lock.hcl state/
+
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \
+TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' |
+xargs) && \
+cp backend.tf.local backend.tf && \
+terraform init -force-copy -lock=false -migrate-state && \
+gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \
+terraform init && \
+terraform destroy -auto-approve  && \
+rm -rf .terraform .terraform.lock.hcl state/
 
      
 
      2. 
 
    
@@ -2075,13 +2102,13 @@ 
    Re-using an Existing Project
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
-${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
-gcloud endpoints services undelete \
-${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
---quiet 2>/dev/null
+
    BACKSTAGE_QS_PREFIX=$(grep environment_name \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+BACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \
+${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F"=" '{print $2}' | xargs)
+gcloud endpoints services undelete \
+${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \
+--quiet 2>/dev/null
 
    
 
 
diff --git a/docs/search/search_index.json b/docs/search/search_index.json
index a9f2c19b..0809ab66 100644
--- a/docs/search/search_index.json
+++ b/docs/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    3.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      4.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable \\\n  iap.googleapis.com \\\n  serviceusage.googleapis.com \\\n  servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Create the Identity Aware Proxy brand
       
      gcloud iap oauth-brands create \\\n  --application_title=\"IAP Secured Backstage\" \\\n  --project=\"${PROJECT_ID}\" \\\n  --support_email=\"${IAP_SUPPORT_EMAIL}$\"\n
       
      3.  
    3.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      4.  
    4.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      This will take approximately 10 minutes to build and push the image.
       
      5.  
    5.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      6.  
    6.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      7.  
    7.  
      Capture the IAP audience
       
      a. In the Cloud Console, navigate to Security > Identity-Aware Proxy
       
      b. Choose Get JWT audience code from the three dot menu on the right side of your Backend Service.
       
      c. The value will be in the format of: /projects/<your_project_number>/global/backendServices/<numeric_id>. Using that value create a new environment variable.
       
      export IAP_AUDIENCE_VALUE=\"<your_iap_audience_value>\"\n
       
      8.  
    8.  
      Deploy the Kubernetes manifests
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\nsed -i \"s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      9.  
    9.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      10.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iap.googleapis.com \\\n  serviceusage.googleapis.com \\\n  servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Setup the Identity Aware Proxy brand
       
      gcloud iap oauth-brands create \\\n  --application_title=\"IAP Secured Backstage\" \\\n  --project=\"${PROJECT_ID}\" \\\n  --support_email=\"${IAP_SUPPORT_EMAIL}\"\n
       
      Capture the brand name in an environment variable, it will be in the format of: projects/[your_project_number]/brands/[your_project_number].
       
      export IAP_BRAND=<your_brand_name>\n
       
      3.  
    3.  
      Using the brand name create the IAP client.
       
      gcloud iap oauth-clients create \\\n  ${IAP_BRAND} \\\n  --display_name=\"IAP Secured Backstage\"\n
       
      Capture the client_id and client_secret in environment variables. For the client_id we only need the last value of the string, it will be in the format of: 549085115274-ksi3n9n41tp1vif79dda5ofauk0ebes9.apps.googleusercontent.com
       
      export IAP_CLIENT_ID=\"<your_client_id>\"\nexport IAP_SECRET=\"<your_iap_secret>\"\n
       
      4.  
    4.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SECRET/${IAP_SECRET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      6.  
    6.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      This will take approximately 10 minutes to build and push the image.
       
      7.  
    7.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      8.  
    8.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      9.  
    9.  
      Perform an initial deployment of Kubernetes resources.
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      10.  
    10.  
      Capture the IAP audience, the Backend Service may take a few minutes to     appear.
       
      a. In the Cloud Console, navigate to Security > Identity-Aware Proxy
       
      b. Verify the IAP option is set to enabled. If not enable it now.
       
      b. Choose Get JWT audience code from the three dot menu on the right side of your Backend Service.
       
      c. The value will be in the format of: /projects/<your_project_number>/global/backendServices/<numeric_id>. Using that value create a new environment variable.
       
      export IAP_AUDIENCE_VALUE=\"<your_iap_audience_value>\"\n
       
      11.  
    11.  
      Redeploy the Kubernetes manifests with the IAP audience
       
      sed -i \"s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g\" deployment.yaml\nkubectl apply -f .\n
       
      12.  
    12.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      13.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index ef7a7c2e..45195732 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -2,78 +2,78 @@
 
     
          https://googlecloudplatform.github.io/platform-engineering/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/code-of-conduct/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/contributing/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/accelerating-migrations/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/automated-password-rotation/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/backstage-quickstart/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudRun/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/WebsiteDemo/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/github-runners-gke/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/cli/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/gcp-sandboxes/
-         2025-07-10
+         2025-07-11
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/sandbox-modules/
-         2025-07-10
+         2025-07-11
     
 
\ No newline at end of file
diff --git a/docs/sitemap.xml.gz b/docs/sitemap.xml.gz
index 1da70047378b4fe4e9cfb0069ffe166874566191..706fbcda1ef76c6dd834857e84ffe8337b8651db 100644
GIT binary patch
literal 437
zcmV;m0ZRTKiwFn+RB&hl|8r?{Wo=<_E_iKh0Nt0tZrd;nhVOlfz;{_mfuY-wxENLx
z=&3-lVaGz28AYfpsw7osZ$H_|irt1`J5t+Sf
z;^3p(HDABns`t%xcd^YB7-Us~Bk!7nNcyPtG)*mgfzc#xk(_3GC^xLnPQ7kFG~LCj
ztU1BSTGlq8?dB|8(l|sHzy{1i3P$!wW9z&ea^HH?KV1r<^AUgwncnI-!LRuY%ZNjF
zxxRd-)>mq?DIsUy2|&LR{9~3h4%1g?x0j%G|@=^S4x
fy0b~uTK}`93QP`vL%&?guVVKDlE~*LS`GjJlq}hX

literal 437
zcmV;m0ZRTKiwFpS0&i#n|8r?{Wo=<_E_iKh0Ns~OYuqpphVS_mi|?|WKHD5m8tGCTff3eLJ7-Um|Bk!7nNP4gJJkKqAfypFpk(_3GC^xLnPTjTdn*L%_
z){@|4t!o?5_DdEnX&j;pU<2kc1tWW;sdZisd1yWApDqQ_`3OLTOmFp);MaVHWx}z)
z>aMO;_eNcICFJZo0q7Tkf6P*f1q%1dVjQ!TSH>2F{E+3f6^&pIR2mJGA$ut8VkHx>
zHr7G_G7GSpyjv`cTnBp=oQ%Mzl3|{aMnx#AfGYS0W1m=zgUV=MEcGq>)g-Ax!kXl9
zgcR`m<38YADfR8*xzEv_zSTA0ZhihCCBkPD_`g&3F-jRzww~$qC*nROsA=&2UK&A}
z%MWE|F$^_JddRU-{I4)9r@}j!unGmI*A(R#%i&nLr#kwEO&kY20#`;0M>D0cc8;$V
f-PxpSt^e6l1ty2Tp
Date: Wed, 16 Jul 2025 18:00:24 +0000
Subject: [PATCH 27/28] Adding feedback from Ameenah.

---
 .../backstage/backstage-quickstart/README.md                 | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/reference-architectures/backstage/backstage-quickstart/README.md b/reference-architectures/backstage/backstage-quickstart/README.md
index 5e2959df..90417bf7 100644
--- a/reference-architectures/backstage/backstage-quickstart/README.md
+++ b/reference-architectures/backstage/backstage-quickstart/README.md
@@ -141,6 +141,7 @@ Management API are enabled.
     sed -i "s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g" ${BACKSTAGE_QS_BASE_DIR}/backend.tf
     sed -i "s/YOUR_PROJECT_ID/${PROJECT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
     sed -i "s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+    sed -i "s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
     sed -i "s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
     sed -i "s/YOUR_IAP_SECRET/${IAP_SECRET}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
     ```
@@ -155,6 +156,10 @@ Management API are enabled.
     rm tfplan
     ```
 
+    Initial run of the Terraform may result in errors due to they way the API
+    services are asyrchonously enabled. Re-running the terraform usually
+    resolves the errors.
+
     This will take a while to create all of the required resources, figure
     somewhere between 15 and 20 minutes.
 

From 76c503be7523e8ef7b8d562d702d8c86730ada37 Mon Sep 17 00:00:00 2001
From: bgood <1019754+bgood@users.noreply.github.com>
Date: Wed, 16 Jul 2025 18:00:51 +0000
Subject: [PATCH 28/28] chore: update documentation site

---
 .../backstage/backstage-quickstart/index.html |   8 +++-
 docs/search/search_index.json                 |   2 +-
 docs/sitemap.xml                              |  38 +++++++++---------
 docs/sitemap.xml.gz                           | Bin 437 -> 438 bytes
 4 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/docs/reference-architectures/backstage/backstage-quickstart/index.html b/docs/reference-architectures/backstage/backstage-quickstart/index.html
index a76b79b1..be1f2bc5 100644
--- a/docs/reference-architectures/backstage/backstage-quickstart/index.html
+++ b/docs/reference-architectures/backstage/backstage-quickstart/index.html
@@ -1894,8 +1894,9 @@ 
    Deploy Backstage
    sed -i "s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g" ${BACKSTAGE_QS_BASE_DIR}/backend.tf
 sed -i "s/YOUR_PROJECT_ID/${PROJECT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
 sed -i "s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-sed -i "s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
-sed -i "s/YOUR_IAP_SECRET/${IAP_SECRET}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+sed -i "s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+sed -i "s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
+sed -i "s/YOUR_IAP_SECRET/${IAP_SECRET}/g" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars
 
    
 
 
  • 
@@ -1906,6 +1907,9 @@ 
    Deploy Backstageterraform apply -input=false tfplan && \
 rm tfplan
 
    • 
+
    Initial run of the Terraform may result in errors due to they way the API
+services are asyrchonously enabled. Re-running the terraform usually
+resolves the errors.
    
 
    This will take a while to create all of the required resources, figure
 somewhere between 15 and 20 minutes.
    
 
diff --git a/docs/search/search_index.json b/docs/search/search_index.json
index 0809ab66..83b428dc 100644
--- a/docs/search/search_index.json
+++ b/docs/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iap.googleapis.com \\\n  serviceusage.googleapis.com \\\n  servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Setup the Identity Aware Proxy brand
       
      gcloud iap oauth-brands create \\\n  --application_title=\"IAP Secured Backstage\" \\\n  --project=\"${PROJECT_ID}\" \\\n  --support_email=\"${IAP_SUPPORT_EMAIL}\"\n
       
      Capture the brand name in an environment variable, it will be in the format of: projects/[your_project_number]/brands/[your_project_number].
       
      export IAP_BRAND=<your_brand_name>\n
       
      3.  
    3.  
      Using the brand name create the IAP client.
       
      gcloud iap oauth-clients create \\\n  ${IAP_BRAND} \\\n  --display_name=\"IAP Secured Backstage\"\n
       
      Capture the client_id and client_secret in environment variables. For the client_id we only need the last value of the string, it will be in the format of: 549085115274-ksi3n9n41tp1vif79dda5ofauk0ebes9.apps.googleusercontent.com
       
      export IAP_CLIENT_ID=\"<your_client_id>\"\nexport IAP_SECRET=\"<your_iap_secret>\"\n
       
      4.  
    4.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SECRET/${IAP_SECRET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      6.  
    6.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      This will take approximately 10 minutes to build and push the image.
       
      7.  
    7.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      8.  
    8.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      9.  
    9.  
      Perform an initial deployment of Kubernetes resources.
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      10.  
    10.  
      Capture the IAP audience, the Backend Service may take a few minutes to     appear.
       
      a. In the Cloud Console, navigate to Security > Identity-Aware Proxy
       
      b. Verify the IAP option is set to enabled. If not enable it now.
       
      b. Choose Get JWT audience code from the three dot menu on the right side of your Backend Service.
       
      c. The value will be in the format of: /projects/<your_project_number>/global/backendServices/<numeric_id>. Using that value create a new environment variable.
       
      export IAP_AUDIENCE_VALUE=\"<your_iap_audience_value>\"\n
       
      11.  
    11.  
      Redeploy the Kubernetes manifests with the IAP audience
       
      sed -i \"s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g\" deployment.yaml\nkubectl apply -f .\n
       
      12.  
    12.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      13.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Platform Engineering on Google Cloud","text":"
    Platform engineering is an emerging practice in organizations to enable cross functional collaboration in order to deliver business value faster. It treats the internal groups; application developers, operators, security, infrastructure admins, etc. as customers and provides them the foundational platforms to accelerate their work. The key goals of platform engineering are providing everything as self-service, golden paths, improved collaboration, abstraction of technical complexities, all of which simplify the software development lifecycle, contributing towards delivering business values to consumers. Platform engineering is more effective in cloud computing as it helps realize the benefits possible on cloud like automation, security, productivity, faster time-to-market.
    "},{"location":"#overview","title":"Overview","text":"
    Google Cloud offers decomposable, elastic, secure, scalable and cost efficient tools built on the guiding principles of platform engineering. With a focus on developer experience and innovation coupled with practices like SRE embedded into the tools, they make a good place to begin your platform journey to empower the developers to enhance their experience and increase their productivity.
     
    This repository contains a collection of guides, examples and design patterns spanning Google Cloud products and best in class OSS tools, which you can use to help build an internal developer platform.
     
    For more information, see Platform Engineering on Google Cloud.
    "},{"location":"#resources","title":"Resources","text":""},{"location":"#design-patterns","title":"Design Patterns","text":"
       
    • Platform Engineering: 5 Implemenation Myths
      •  
    • Business continuity planning for CI/CD
      •  
    "},{"location":"#research-papers-and-white-papers","title":"Research papers and white papers","text":"
       
    • Google Cloud ESG Strategic Guide: Discover the power of platform   engineering
      •  
    • Mastering Platform Engineering: Key Insights from Industry   Experts
      •  
    "},{"location":"#guides-and-building-blocks","title":"Guides and Building Blocks","text":""},{"location":"#manage-developer-environments-at-scale","title":"Manage Developer Environments at Scale","text":"
       
    • Backstage Plugin for Cloud Workstations
      •  
    "},{"location":"#self-service-and-automation-patterns","title":"Self-service and Automation patterns","text":"
       
    • Automatic password rotation
      •  
    "},{"location":"#run-third-party-cicd-tools-on-google-cloud-infrastructure","title":"Run third-party CI/CD tools on Google Cloud infrastructure","text":"
       
    • Host GitHub Actions Runners on GKE
      •  
    "},{"location":"#enterprise-change-management","title":"Enterprise change management","text":"
       
    • Integrate Cloud Deploy with enterprise change management   systems
      •  
    "},{"location":"#end-to-end-examples","title":"End-to-end Examples","text":"
       
    • Enterprise Application Blueprint - Deploys an   internal developer platform that enables cloud platform teams to provide a   managed software development and delivery platform for their organization's   application development groups. EAB builds upon the infrastructure foundation   deployed using the Enterprise Foundation   blueprint.
      •  
    • Software Delivery Blueprint - An opinionated   approach using platform engineering to improve software delivery, specifically   for Infrastructure admins, Operators, Security specialists, and Application   developers. It utilizes GitOps and self-service workflows to enable consistent   infrastructure, automated security policies, and autonomous application   deployment for developers.
      •  
    "},{"location":"#usage-disclaimer","title":"Usage Disclaimer","text":"
    Copy any code you need from this repository into your own project.
     
    Warning: Do not depend directly on the samples in this repository. Breaking changes may be made at any time without warning.
    "},{"location":"#contributing-changes","title":"Contributing changes","text":"
    Entirely new samples are not accepted. Bugfixes are welcome, either as pull requests or as GitHub issues.
     
    See CONTRIBUTING.md for details on how to contribute.
    "},{"location":"#licensing","title":"Licensing","text":"
    Copyright 2024 Google LLC Code in this repository is licensed under the Apache 2.0. See LICENSE.
    "},{"location":"code-of-conduct/","title":"Code of Conduct","text":""},{"location":"code-of-conduct/#our-pledge","title":"Our Pledge","text":"
    In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
    "},{"location":"code-of-conduct/#our-standards","title":"Our Standards","text":"
    Examples of behavior that contributes to creating a positive environment include:
     
       
    • Using welcoming and inclusive language
      •  
    • Being respectful of differing viewpoints and experiences
      •  
    • Gracefully accepting constructive criticism
      •  
    • Focusing on what is best for the community
      •  
    • Showing empathy towards other community members
      •  
     
    Examples of unacceptable behavior by participants include:
     
       
    • The use of sexualized language or imagery and unwelcome sexual attention or   advances
      •  
    • Trolling, insulting/derogatory comments, and personal or political attacks
      •  
    • Public or private harassment
      •  
    • Publishing others' private information, such as a physical or electronic   address, without explicit permission
      •  
    • Other conduct which could reasonably be considered inappropriate in a   professional setting
      •  
    "},{"location":"code-of-conduct/#our-responsibilities","title":"Our Responsibilities","text":"
    Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
     
    Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
    "},{"location":"code-of-conduct/#scope","title":"Scope","text":"
    This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
     
    This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community.
    "},{"location":"code-of-conduct/#conflict-resolution","title":"Conflict Resolution","text":"
    We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project\u2019s code of conduct.
     
    If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe.
     
    Reports should be directed to [PROJECT STEWARD NAME(s) AND EMAIL(s)], the Project Steward(s) for [PROJECT NAME]. It is the Project Steward\u2019s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email opensource@google.com.
     
    We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice.
    "},{"location":"code-of-conduct/#attribution","title":"Attribution","text":"
    This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
    "},{"location":"contributing/","title":"How to Contribute","text":"
    We'd love to accept your patches and contributions to this project.
    "},{"location":"contributing/#before-you-begin","title":"Before you begin","text":""},{"location":"contributing/#sign-our-contributor-license-agreement","title":"Sign our Contributor License Agreement","text":"
    Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project.
     
    If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again.
     
    Visit https://cla.developers.google.com/ to see your current agreements or to sign a new one.
    "},{"location":"contributing/#review-our-community-guidelines","title":"Review our Community Guidelines","text":"
    This project follows Google's Open Source Community Guidelines.
    "},{"location":"contributing/#contribution-process","title":"Contribution process","text":""},{"location":"contributing/#code-reviews","title":"Code Reviews","text":"
    All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult GitHub Help for more information on using pull requests.
    "},{"location":"contributing/#development-guide","title":"Development guide","text":"
    This document contains technical information to contribute to this repository.
    "},{"location":"contributing/#site","title":"Site","text":"
    This repository includes scripts and configuration to build a site using Material for MkDocs:
     
       
    • config/mkdocs: MkDocs configuration files
      •  
    • scripts/run-mkdocssh: script to build the site
      •  
    • .github/workflows/documentation.yaml: GitHub Actions workflow that builds   the site, and pushes a commit with changes on the current branch.
      •  
    "},{"location":"contributing/#build-the-site","title":"Build the site","text":"
    To build the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh\n
    "},{"location":"contributing/#preview-the-site","title":"Preview the site","text":"
    To preview the site, run the following command from the root of the repository:
     
    scripts/run-mkdocs.sh \"serve\"\n
    "},{"location":"contributing/#linting-and-formatting","title":"Linting and formatting","text":"
    We configured several linters and formatters for code and documentation in this repository. Linting and formatting checks run as part of CI workflows.
     
    Linting and formatting checks are configured to check changed files only by default. If you change the configuration of any linter or formatter, these checks run against the entire repository.
     
    To run linting and formatting checks locally, you do the following:
     
    scripts/lint.sh\n
     
    To automatically fix certain linting and formatting errors, you do the following:
     
    LINTER_CONTAINER_FIX_MODE=\"true\" scripts/lint.sh\n
    "},{"location":"reference-architectures/accelerating-migrations/","title":"Accelerate migrations through platform engineering golden paths","text":"
    This document helps you adopt platform engineering by designing a process to onboard and migrate your existing applications to use your internal developer platform (IDP). It also provides guidance to help you evaluate the opportunity to design a platform engineering process, and to explore how it might function. Google Cloud provides tools, products, guidance, and professional services to help you adopt platform engineering in your environments.
     
    This document is aimed at the following personas:
     
       
    • Application developers, to help them understand how to refactor and   modernize applications to onboard and migrate them on the IDP.
      •  
    • Application operator, to help them understand how to integrate the   application with the IDP's observability mechanisms.
      •  
    • Platform administrators, to highlight possible platform enhancements to   ease onboarding and migration of applications.
      •  
    • Database administrators, to help them migrate from self-managed databases   to managed database services.
      •  
    • Security specialists, to outline possible security challenges and benefit   from IDP's security solutions.
      •  
     
    The Cloud Native Computing Foundation defines a golden path as an integrated bundle of templates and documentation for rapid project development. Designing and developing golden paths can help facilitate the onboarding and the migration of existing applications to your IDP. When you use a golden path, your development and operations teams can take advantage of benefits like the following:
     
       
    • Streamlined, self-service development and deployment processes.
      •  
    • Ready-to-use infrastructure, and templates for your projects.
      •  
    • Observability instrumentation.
      •  
    • Extensive reference documentation.
      •  
     
    Onboarding and migrating existing applications to the IDP can let you experience the benefits of adopting platform engineering gradually and incrementally in your organization, without spending effort on large scale migration projects.
     
    To migrate applications and onboard them to the IDP, we recommend that you design an application onboarding and migration process. This document describes a reference application onboarding and migration process. We recommend that you tailor the process to your requirements and your IDP.
     
    If you're migrating your applications from your on-premises environment or from another cloud provider to Google Cloud, the application onboarding and migration process can help you to accelerate your migration. In that scenario, the teams that are managing the migration can refer to well-established golden paths, instead of having to design their own migration processes and project templates.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-process","title":"Application onboarding and migration process","text":"
    The goal of the application onboarding and migration process is to get an application on the IDP. After you onboard and migrate the application to the IDP, your teams can benefit from using the IDP. When you use an IDP, you can focus on providing business value for the application, rather than spending effort on ad-hoc processes and operations.
     
    To manage the complexity of the application onboarding and migration process, we recommend that you design the process in the following phases:
     
       
    1. Intake the application onboarding and migration request.
      2.  
    2. Assess the application to onboard and migrate.
      3.  
    3. Set up and eventually extend the IDP to accommodate the needs of the    application to onboard and migrate.
      4.  
    4. Onboard and migrate the application.
      5.  
    5. Optimize the application.
      6.  
     
    The high-level structure of this process matches the Google Cloud migration path. In this case, you follow the migration path to onboard and migrate existing applications on the IDP.
     
    To ensure that the application onboarding and migration is on the right track, we recommend that you design validation checkpoints for each phase of the process, rather than having a single acceptance testing task. Having validation checkpoints for each phase helps you to promptly detect issues as they arise, rather than when you are close to the end of the migration.
     
    Even when following a phased process, onboarding and migrating complex applications to the IDP might require a significant effort, and it might pose risks. To manage the effort and the risks of onboarding and migrating complex applications to the IDP, you can follow the onboarding and migration process iteratively, by migrating parts of the application on each iteration. For example, if an application is composed of multiple components, you can onboard and migrate one component for each iteration of the process.
     
    To reduce toil, we recommend that you thoroughly document the application onboarding and migration process, and make it as self-service as possible, in line with platform-engineering principles.
     
    In this document, we assume that the onboarding and migration process involves three teams:
     
       
    • Application onboarding and migration team: the team that's responsible for   onboarding and migrating the application on the IDP.
      •  
    • Application development and operations team: the team that's responsible   for developing and operating the application.
      •  
    • IDP team: the team that's responsible for developing and operating the   IDP.
      •  
     
    The following sections describe each phase of the application onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request","title":"Intake the onboarding and migration request","text":"
    The first phase of the application onboarding and migration process is to intake the request to onboard and migrate the application. The request process is the following:
     
       
    1. The application onboarding and migration team files the onboarding and    migration request.
      2.  
    2. The IDP receives the request, and it recommends existing golden paths.
      3.  
    3. If the IDP can't suggest an existing golden path, the IDP forwards the    request to the team that manages the IDP for further evaluation.
      4.  
     
    We recommend that you keep this phase as light as possible by using a form or a guided, self-service process. For example, you can include migration guidance in the IDP documentation so that development teams can review it and prepare for the migration. You can also implement automated checks in your IDP to give initial feedback to development teams about potential migration blockers and issues.
     
    To assist and offer consultation to the teams that filed or intend to file an application onboarding and migration request, we recommend that the team that manages the IDP establish communication channels to offer assistance to other teams. For example, the team that manages the IDP might set up dedicated discussion groups, chat rooms, and office hours where they can offer help and answer questions about the IDP. To help with onboarding and migration of complex applications and to facilitate communications, you can also attach a member of the team that manages the IDP to the application team while the migration is in progress.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration","title":"Plan application onboarding and migration","text":"
    As part of this phase, we recommend that the application onboarding and migration team starts drafting an onboarding and migration plan, even if the team doesn't have all of the data points to fully define it. When the team progresses through the assessment phase, they will gather information to finalize and validate the plan.
     
    To manage the complexity of the migration plan, we recommend that you decompose it across the following sub-tasks:
     
       
    • Define the timelines for the onboarding and migration process, and any   intermediate milestones, according to the requirements of the application   onboarding and migration. For example, you might develop a countdown plan that   lists all of the tasks that are required to complete the application   onboarding and migration, along with responsibilities and estimated duration.
      •  
    • Define a responsibility assignment (RACI) matrix to clearly outline who is   responsible for each phase and task that composes the onboarding and migration   project.
      •  
    • Monitor the onboarding and migration process, to gather data so that you can   optimize the process. For example, you might gather data about how much time   you spend on each phase and on each task of the onboarding and migration   process. You might also gather data about the most common blockers and issues   that you experience during the process.
      •  
     
    Developing a comprehensive onboarding and migration plan is crucial to the success of the application onboarding and migration process. Having a plan helps you to define clear deadlines, assign responsibilities, and deal with unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-the-application","title":"Assess the application","text":"
    The second phase of the application onboarding and migration process is to follow up on the intake request by assessing the application to onboard and migrate to the IDP. The goal of this assessment phase is to produce the following artifacts:
     
       
    • Data about the architecture of the application and its deployment and   operational processes.
      •  
    • Plans to migrate the application and onboard it to the IDP.
      •  
     
    These outputs of the assessment phase help you to plan and complete the migration. The outputs also help you to scope the enhancements that the IDP needs to support the application, and to increase the velocity of future migrations.
     
    To manage the complexity of the assessment phase, we recommend that you decompose it into the following steps:
     
       
    1. Review the application design.
      2.  
    2. Review application dependencies.
      3.  
    3. Review continuous integration and continuous deployment (CI/CD) processes.
      4.  
    4. Review data persistence and data management requirements.
      5.  
    5. Review FinOps requirements.
      6.  
    6. Review compliance requirements.
      7.  
    7. Review the application team practices.
      8.  
    8. Assess application refactoring and the IDP.
      9.  
    9. Finalize the application onboarding and migration plan.
      10.  
     
    The preceding steps are described in the following sections. For more information about assessing applications and defining migration plans, see Migrate to Google Cloud: Assess and discover your workloads.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design","title":"Review the application design","text":"
    To gather a comprehensive understanding about the design of the application, we recommend that you complete a thorough assessment of the following aspects of the application:
     
       
    • Application source code:
         
      • Ensure that the source code of the application is available, and that you   can access it.
        •  
      • Gather information about how many repositories you're using to store the   source code of the application and the structure of the repositories.
        •  
      • Review the deployment descriptors that you're using for the application.
        •  
      • Review any code that's responsible for handling provisioning and   configuration of the necessary infrastructure.
        •  
       
      •  
    • Deployable artifacts: Gather information about the deployable artifacts   that you're using to package and deploy your application, such as container   images, packages, and the repositories that you're using to store them.
      •  
    • Configuration injection: Assess how you're injecting configuration inside   deployable artifacts. For example, gather information about how you're   distributing environment- and deployment-specific configuration to your   application.
      •  
    • Security requirements: Collect data about the security requirements and   processes that you have in place for the application, such as vulnerability   scanning, binary authorization, bills of materials verification, attestation,   and secret management.
      •  
    • Identity and access management: Gather information about how your   application handles identity and access management, and the roles and   permissions that your application assumes for its users.
      •  
    • Observability requirements:
         
      • Assess your application's observability requirements, in terms of   monitoring, logging, tracing and alerting.
        •  
      • Gather information about any service level objectives (SLOs) that are in   place for the application.
        •  
       
      •  
    • Availability and reliability requirements:
         
      • Gather information about the availability and reliability requirements of   the application.
        •  
      • Define the   failure modes   that the application supports.
        •  
       
      •  
    • Network and connectivity requirements:
         
      • Assess the network requirements for your application, such as IP address   space, DNS names, load balancing and failover mechanisms.
        •  
      • Gather information about any connectivity requirements to other   environments, such as on-premises and third-party ones.
        •  
      • Gather information about any other services that your application might   need, such as API gateways and service meshes.
        •  
       
      •  
    • Statefulness: Develop a comprehensive understanding of how the application   handles stateful data, if any, and where the application stores data. For   example, gather information about persistent stateful data, such as data   stored in databases, object storage services, persistent disks, and transient   data like caches.
      •  
    • Runtime environment requirements: Gather information about the runtime   requirements of the application, such as any dependency the application needs   to run. For example, your application might need certain libraries, or have   platform or API dependencies.
      •  
    • Development tools and environments. Assess the development tools and   environments that developers use to support and evolve your application, such   as integrated development environments (IDEs) along with any IDE extensions,   the configuration of their development workstations, and any development   environment they use to support their work.
      •  
    • Multi-tenancy requirements. Gather information about any multi-tenancy   requirements for the application.
      •  
     
    Understanding the application architecture helps you to design and implement an effective onboarding and migration process for your application. It also helps you anticipate issues and potential problems that might arise during the migration. For example, if the architecture of your application to onboard and migrate to the IDP isn't compatible with your IDP, you might need to spend additional effort to refactor the application and enhance the IDP.
     
       
    •  
      •  
     
    The application to onboard and migrate to the IDP might have dependencies on systems and data that are outside the scope of the application. To understand these dependencies, we recommend that you gather information about any reliance of your application on external systems and data, such as databases, datasets, and APIs. After you gather information, you classify the dependencies in order of importance and criticality. For example, your application might need access to a database to store persistent data, and to external APIs to integrate with to provide critical functionality to users, while it might have an optional dependency on a caching system.
     
    Understanding the dependencies of your application on external systems and data is crucial to plan for continued access to these dependencies during and after the migration.
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies","title":"Review application dependencies","text":""},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes","title":"Review CI/CD processes","text":"
    After you review the application design and its dependencies, we recommend that you refine the assessment about your application's deployable artifacts by reviewing your application's CI/CD processes. These processes usually let you build the artifacts to deploy the application and let you deploy them in your runtime environments. For example, you refine the assessment by answering questions about these CI/CD processes, such as the following:
     
       
    • Which systems are you using as part of the CI/CD workflows to build and deploy   your application?
      •  
    • Where do you store the deployable artifacts that you build for the   application?
      •  
    • How frequently do you deploy the application?
      •  
    • What are your deployment processes like? For example, are you using any   advanced deployment methodology, such as canary deployments, or blue-green   deployments?
      •  
    • Do you need to migrate the deployable artifacts that you previously built for   the application?
      •  
     
    Understanding how the application's CI/CD processes work helps you evaluate whether your IDP can support these CI/CD processes as is, or if you need to enhance your IDP to support them. For example, if your application has a business-critical requirement on a canary deployment process and your IDP doesn't support it, you might need to factor in additional effort to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements","title":"Review data persistence and data management requirements","text":"
    By completing the previous tasks of the assessment phase, you gathered information about the statefulness of the application and about the systems that the application uses to store persistent and transient data. In this section, you refine the assessment to develop a deeper understanding of the systems that the application uses to store stateful data. We recommend that you gather information on data persistence and data management requirements of your application. For example, you refine the assessment by answering questions such as the following:
     
       
    • Which systems does the application use to store persistent data, such as   databases, object storage systems, and persistent disks?
      •  
    • Does the application use any system to store transient data, such as caches,   in-memory databases, and temporary data disks?
      •  
    • How much persistent and transient data does the application produce?
      •  
    • Do you need to migrate any data when you onboard and migrate the application   to the IDP?
      •  
    • Does the application depend on any data transformation workflows, such as   extract, transform, and load (ETL)   jobs?
      •  
     
    Understanding your application's data persistence and data management requirements helps you to ensure that your IDP and your production environment can effectively support the application. This understanding can also help you determine whether you need to enhance the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements","title":"Review FinOps requirements","text":"
    As part of the assessment of your application, we recommend that you gather data about the FinOps requirements of your application, such as budget control and cost management, and evaluate whether your IDP supports them. For example, the application might require certain mechanisms to control spending and manage costs, eventually sending alerts. The application might also require mechanisms to completely stop spending when it reaches a certain budget limit.
     
    Understanding your application's FinOps requirements helps you to ensure that you keep your application costs under control. It also helps you to establish proper cost attribution and cost optimization practices.
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements","title":"Review compliance requirements","text":"
    The application to onboard and migrate to the IDP and its runtime environment might have to meet compliance requirements, especially in regulated industries. We recommend that you assess the compliance requirements of the application, and evaluate if the IDP already supports them. For example, the application might require isolation from other workloads, or it might have data locality requirements.
     
    Understanding your application's compliance requirements helps you to scope the necessary refactoring and enhancements for your application and for the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-team-practices","title":"Review the application team practices","text":"
    After you review the application, we recommend that you gather information about team practices and the methodologies for developing and operating the application. For example, the team might already have adopted DevOps principles, they might be already implementing Site Reliability Engineering (SRE), or they might be already familiar with platform engineering and with the IDP.
     
    By gathering information about the team that develops and operates the application to migrate, you gain insights about the experience and the maturity of that team. You also learn whether there's a need to spend effort to train team members to proficiently use the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp","title":"Assess application refactoring and the IDP","text":"
    After you gather information about the application, its development and operation teams, and its requirements, you evaluate the following:
     
       
    • Whether the application will work as-is if migrated and onboarded to the IDP.
      •  
    • Whether the IDP can support the application to onboard and migrate.
      •  
     
    The goal of this task is to answer the following questions:
     
       
    1. Does the application need any refactoring to onboard and migrate it to the    IDP?
      2.  
    2. Are there any new services or processes that the IDP should offer to migrate    the application?
      3.  
    3. Does the IDP meet the compliance and regulatory requirements that the    application requires?
      4.  
     
    By answering these questions, you focus on evaluating potential onboarding and migration blockers. For example, you might experience the following onboarding and migration blockers:
     
       
    • If the application doesn't meet the observability or configurability   requirements of the IDP, you might need to enhance the application to meet   those requirements. For example, you might need to refactor the application to   expose a set of metrics on a given endpoint, or to accept configuration   injection as supported by the IDP.
      •  
    • If the application relies on dependencies that suffer from known security   vulnerabilities, you might need to spend effort to update vulnerable   dependencies or to mitigate the vulnerabilities.
      •  
    • If the application has a critical dependency on a service that the IDP doesn't   offer, you might need to refactor the application to avoid that dependency, or   you might consider extending the IDP to offer that service.
      •  
    • If the application depends on a self-managed service that the IDP also offers   as a managed service, you might need to refactor the application to migrate   from the self-managed service to the managed service.
      •  
     
    The application development and operations team is responsible for the application refactoring tasks.
     
    When you scope the eventual enhancements that the IDP needs to support the application, we recommend that you frame these enhancements in the broader vision that you have for the IDP, and not as a standalone exercise. We also recommend that you consider your IDP as a product for which you should develop a path to success. For example, if you're considering adding a new service to the IDP, we recommend that you evaluate how that service fits in the path to success for your IDP, in addition to the technical feasibility of the initiative.
     
    By assessing the refactoring effort that's required to onboard and migrate the application, you develop a comprehensive understanding of the tasks that you need to complete to refactor the application and how you need to enhance the IDP to support the application.
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan","title":"Finalize the application onboarding and migration plan","text":"
    To complete the assessment phase, you finalize the application onboarding and migration plan with consideration of the data that you gathered. To finalize the plan, you do the following:
     
       
    • Develop a rollback strategy for each task to recover from unanticipated issues   that might arise during the application onboarding and migration.
      •  
    • Define criteria to safely retire the environment where the application runs   before you onboard and migrate it to the IDP. For example, you might require   that the application works as expected after onboarding and migrating it to   the IDP before you retire the old environment.
      •  
    • Validate the migration plan to help you avoid unanticipated issues. For more   information about validating a migration plan, see   Migrate to Google Cloud: Best practices for validating a migration plan.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp","title":"Set up the IDP","text":"
    After you complete the assessment phase, you use its outputs to:
     
       
    1. Enhance the IDP by adding missing features and services.
      2.  
    2. Configure the IDP to support the application.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp","title":"Enhance the IDP","text":"
    During the assessment phase, you scope any enhancements to the IDP that it needs to support the application and how those enhancements fit in your plans for the IDP. By completing this task, you design and implement the enhancements. For example, you might need to enhance the IDP as follows:
     
       
    • Add services to the IDP, in case the application has critical dependencies on   such services, and you can't refactor the application. For example, if the   application needs an in-memory caching service and the IDP doesn't offer that   service yet, you can add a data store like Cloud Memorystore to the IDP's   portfolio of services.
      •  
    • Meet the application's compliance requirements. For example, if the   application requires that its data must reside in certain geographic regions   and the IDP doesn't support deploying resources in those regions, you need to   enhance the IDP to support those regions.
      •  
    • Support further configurability and observability to cover the application's   requirements. For example, if the application requires monitoring certain   metrics, and the IDP doesn't support those metrics, you might enhance the   configuration injection and observability services that the IDP provides.
      •  
     
    By enhancing the IDP to support the application, you unblock the migration. You also help streamline processes for onboarding and migration projects for other applications that might need those IDP enhancements.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp","title":"Configure the IDP","text":"
    After you enhance the IDP, if needed, you configure it to provide the resources that the application needs. For example, you configure the following IDP services for the application, or a subset of services:
     
       
    • Foundational services, such as   Google Cloud folders and projects,   Identity and access management (IAM), network connectivity, Virtual Private   Cloud (VPC), and DNS zones and records.
      •  
    • Compute resources, such as Google Kubernetes Engine clusters, and Cloud Run   services.
      •  
    • Data management resources, such as Cloud SQL databases and DataFlow jobs.
      •  
    • Application-level services, such as API gateways, Cloud Service Mesh, and   Cloud Storage buckets.
      •  
    • Application delivery services, such as source code repositories, and Artifact   Registry repositories.
      •  
    • AI/ML services, such as Vertex AI.
      •  
    • Messaging and event processing services, such as Cloud Pub/Sub and Eventarc.
      •  
    • Instrument observability services, such as Cloud Operations Suite.
      •  
    • Security and secret management services, such as Cloud Key Management Service   and Secret Manager.
      •  
    • Cost management and FinOps services, such as Cloud Billing.
      •  
     
    By configuring the IDP, you prepare it to host the application that you want to onboard and migrate.
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application","title":"Onboard and migrate the application","text":"
    In this phase, you onboard and migrate the application to the IDP by completing the following tasks:
     
       
    1. Refactor the application to apply the changes that are necessary to onboard    and migrate it on the IDP.
      2.  
    2. Configure CI/CD workflows for the application and deploy the application in    the development environment.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
    4. Perform acceptance testing.
      5.  
    5. Migrate data from the source environment to the production environment.
      6.  
    6. Promote the application from the staging environment to the production    environment and ensure the application's operational readiness.
      7.  
    7. Perform the cutover from the source environment.
      8.  
     
    By completing the preceding tasks, you onboard and migrate the application to the IDP. The following sections describe these tasks in more detail.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application","title":"Refactor the application","text":"
    In the assessment phase, you scoped the refactoring that your application needs in order to onboard and migrate it to the IDP. By completing this task, you design and implement the refactoring. For example, you might need to refactor your application in the following ways in order to meet the IDP's requirements:
     
       
    • Support the IDP's configuration mechanisms. For example, the IDP might   distribute configuration to applications using environment variables or   templated configuration files.
      •  
    • Refactor the existing application observability mechanisms, or implement them   if there are none, to meet the IDP's observability requirements. For example,   the IDP might require that applications expose a defined set of metrics to   observe.
      •  
    • Update the application's dependencies that suffer from known vulnerabilities.   For example, you might need to update operating system packages and software   libraries that suffer from known vulnerabilities.
      •  
    • Avoid dependencies on services that the IDP doesn't offer. For example, if the   application depends on an object storage service that the IDP doesn't support,   you might need to refactor the application to migrate to a supported object   storage service, such as Cloud Storage.
      •  
    • Migrate from self-managed services to IDP services. For example, if your   application depends on a self-managed database, you might refactor it to use a   database service that the IDP offers, such as Cloud SQL.
      •  
     
    By refactoring the application, you prepare it to onboard and migrate it on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows","title":"Configure CI/CD workflows","text":"
    After you refactor the application, you do the following:
     
       
    1. Configure CI/CD workflows to deploy the application.
      2.  
    2. Optionally migrate deployable artifacts from the source environment.
      3.  
    3. Deploy the application in the development environment.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-to-deploy-the-application","title":"Configure CI/CD workflows to deploy the application","text":"
    To build deployable artifacts and deploy them in your runtime environments, we recommend that you avoid manual processes. Instead of manual processes, configure CI/CD workflows by using the application delivery services that the IDP provides and store deployable artifacts in IDP-managed artifact repositories. For example, you can configure CI/CD workflows by using the following methods:
     
       
    1. Configure Cloud Build to    build container images and store them in Artifact Registry.
      2.  
    2. Configure a    Cloud Deploy pipeline to automate delivery of your application.
      3.  
     
    When you build the CI/CD workflows for your environment, consider how many runtime environments the IDP supports. For example, the IDP might support different runtime environments that are isolated from each other such as the following:
     
       
    • Development environment: for development and testing.
      •  
    • Staging environment: for validation and acceptance testing.
      •  
    • Production environment: for your production workloads.
      •  
     
    If the IDP supports multiple runtime environments for the application, you need to configure the CI/CD workflows for the application to support promoting the application's deployable artifact. You should plan for promoting the application from development to staging, and then from staging to production.
     
    When you promote the application from one environment to the next environment, we recommend that you avoid rebuilding the application's deployable artifacts. Rebuilding creates new artifacts, which means that you would be deploying something different than what you tested and validated.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-deployable-artifacts-from-the-source-environment","title":"Migrate deployable artifacts from the source environment","text":"
    If you need to support rolling back to previous versions of the application, you can migrate previous versions of the deployable artifacts that you built for the application from the source environment to an IDP-managed artifact repository. For example, if your application is containerized, you can migrate the container images that you built to deploy the application to Artifact Registry.
    "},{"location":"reference-architectures/accelerating-migrations/#deploy-the-application-in-the-development-environment","title":"Deploy the application in the development environment","text":"
    After configuring CI/CD workflows to build deployable artifacts for the application and to promote them from one environment to another, you deploy the application in the development environment using the CI/CD workflows that you configured.
     
    By using CI/CD workflows to build deployable artifacts and deploy the application, you avoid manual processes that are less repeatable and more prone to errors. You also validate that the CI/CD workflows work as expected.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-development-to-staging","title":"Promote from development to staging","text":"
    To promote your application from the development environment to the staging environment, you do the following:
     
       
    1. Test the application and verify that it works as expected.
      2.  
    2. Fix any unanticipated issues.
      3.  
    3. Promote the application from the development environment to the staging    environment.
      4.  
     
    By promoting the application from the development environment to the staging environment, you accomplish the following:
     
       
    • Complete a first set of validation tasks.
      •  
    • Polish the application by fixing unanticipated issues.
      •  
    • Enable your teams for broader and deeper functional and non-functional testing   of the application in the staging environment.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing","title":"Perform acceptance testing","text":"
    After you promote the application to your staging environment, you perform extensive acceptance testing for both functional and non-functional requirements. When you perform acceptance testing, we recommend that you validate that the user journeys and the business processes that the application implements are working properly in situations that resemble real-world usage scenarios. For example, when you perform acceptance testing, you can do the following:
     
       
    • Evaluate whether the application works on data that's similar in scope and   size to production data. For example, you can periodically populate your   staging environment with data from the production environment.
      •  
    • Ensure that the application can handle production-like traffic. For example,   you can mirror production traffic and direct it to the application in the   staging environment.
      •  
    • Validate that the application works as designed under degraded conditions. For   example, in your staging environment you can artificially cause outages and   break connectivity to other systems and evaluate whether the application   respects its failure mode. This testing lets you verify that the application   recovers after you terminate the outage, and that it restores connectivity.
      •  
    • Verify that the application, the staging environment, and the production   environment meet your compliance requirements, such as locality restrictions,   licensing, and auditability.
      •  
     
    Acceptance testing helps you ensure that the application works as expected in an environment that resembles the production environment, and helps you identify unanticipated issues.
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data","title":"Migrate data","text":"
    After you complete acceptance testing for the application, you migrate data from the source environment to IDP-managed services such as the following:
     
       
    • Migrate data from databases in the source environment to IDP-managed   databases.
      •  
    • Migrate data from object storage services to IDP-managed object storage   services.
      •  
     
    To migrate data from your source environment to IDP-managed services, you can choose approaches like the following, depending on your requirements:
     
       
    • Scheduled maintenance: Also called one-time migration or offline   migration, with this approach you migrate data during scheduled maintenance   when your application can afford the downtime represented by a planned   cut-over window.
      •  
    • Continuous replication: Also called continuous migration or online   migration, continuous replication builds the scheduled maintenance approach   to reduce the cut-over window size. The size reduction is possible because you   provide a continuous replication mechanism after the initial data copy and   validation.
      •  
    • Y (writing and reading): Also called parallel migration, this approach   is suitable for applications that cannot afford the downtime that's   represented by a cut-over window, even if small. By following this approach,   you refactor the application to write data to both the source environment and   to IDP-managed services. Then, when you're ready to migrate, you switch to   reading data from IDP-managed services.
      •  
    • Data-access microservice: This approach builds on the Y (writing and   reading) approach by centralizing data read and write operations in a scalable   microservice.
      •  
     
    Each of the preceding approaches focuses on solving specific issues, and there's no approach that's inherently better than the others. For more information about migrating data to Google Cloud and choosing the best data migration approach for your application, see Migrate to Google Cloud: Transfer your large datasets.
     
    I your data is stored in services managed by other cloud providers, see the following resources:
     
       
    • Migrate from AWS to Google Cloud: Get started
      •  
    • Migrate from Azure to Google Cloud: Get started
      •  
     
    Migrating data from one environment to another is a complex task. If you think that the data migration is too complex to handle it as part of the application onboarding and migration process, you might consider migrating data as part of a dedicated migration project.
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production","title":"Promote from staging to production","text":"
    After you complete data migration and acceptance testing, you promote the application to the production environment. To complete this task, you do the following:
     
       
    1. Promote the application from the staging environment to the production    environment. The process is similar to when you promoted the application from    the development environment to the staging environment: you use the    IDP-managed CI/CD workflows that you configured for the application to    promote it from the staging environment to the production environment.
      2.  
    2. Ensure the application's operational readiness. For example, to help you    avoid performance issues if the application requires a cache, ensure that the    cache is properly initialized.
      3.  
    3. Fix any unanticipated issues.
      4.  
     
    When you check the application's operational readiness before you promote it from the staging environment to the production environment, you ensure that the application is ready for the production environment.
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover","title":"Perform the cutover","text":"
    After you promote the application to the production environment and ensure that it works as expected, you configure the production environment to gradually route requests for the application to the newly promoted application release. For example, you can implement a canary deployment strategy that uses Cloud Deploy.
     
    After you validate that the application continues to work as expected while the number of requests to the newly promoted application increases, you do the following:
     
       
    1. Configure your production environment to route all of the requests to your    newly promoted application.
      2.  
    2. Retire the application in the source environment.
      3.  
     
    Before you retire the application in the source environment, we recommend that you prepare backups and a rollback plan. Doing so will help you handle unanticipated issues that might force you to go back to using the source environment.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application","title":"Optimize the application","text":"
    Optimization is the last phase of the onboarding and migration process. In this phase, you iterate on optimization tasks until your target environment meets your optimization requirements. For each iteration, you do the following:
     
       
    1. Assess your current environment, teams, and optimization loop.
      2.  
    2. Establish your optimization requirements and goals.
      3.  
    3. Optimize your environment and your teams.
      4.  
    4. Tune the optimization loop.
      5.  
     
    You repeat the preceding sequence until you achieve your optimization goals.
     
    For more information about optimizing your Google Cloud environment, see Migrate to Google Cloud: Optimize your environment and Google Cloud Architecture Framework: Performance optimization.
     
    The following sections integrate the considerations in Migrate to Google Cloud: Optimize your environment.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements","title":"Establish your optimization requirements","text":"
    Optimization requirements help you to narrow the scope of the current optimization iteration. To establish your optimization requirements for the application, start by considering the following aspects:
     
       
    • Security, privacy, and compliance: help you enhance the security posture   of your environment.
      •  
    • Reliability: help you improve the availability, scalability, and   resilience of your environment.
      •  
    • Cost optimization: help you to optimize the resource consumption and   resulting cost of your environment.
      •  
    • Operational efficiency: help you maintain and operate your environment   efficiently.
      •  
    • Performance optimization: help you optimize the performance of the   workloads that are deployed in your environment.
      •  
     
    For each aspect, we recommend that you establish your optimization requirements for the application. Then, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
     
    After you realize the optimization requirements for the application, you completed the onboarding and migration process for the application.
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-onboarding-and-migration-process-and-the-idp","title":"Optimize the onboarding and migration process and the IDP","text":"
    After you onboard and migrate the application, you use the data that you gathered about the process and about the IDP to refine and optimize the process. Similarly to the optimization phase for your application, you complete the tasks that are described in the optimization phase, but with a focus on the onboarding and migration process and on the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#establish-your-optimization-requirements-for-the-idp","title":"Establish your optimization requirements for the IDP","text":"
    To narrow down the scope to optimize the onboarding and migration process, and the IDP, you establish optimization requirements according to data you gather while running through the process. For example, during the onboarding and migration of an application, you might face unanticipated issues that involve the process and the IDP, such as:
     
       
    • Missing documentation about the process
      •  
    • Missing data to complete tasks
      •  
    • Tasks that take too much time to complete
      •  
    • Unclear responsibility mapping
      •  
    • Suboptimal information sharing
      •  
    • Lack of stakeholder engagement
      •  
    • IDP not supporting one or more application use cases
      •  
    • IDP lacking support for one or more services
      •  
    • IDP lacking support for the application's multi-tenancy requirements
      •  
    • IDP not working as expected and documented
      •  
    • Absence of golden paths to for the application
      •  
     
    To address the issues that arise while you're onboarding and migrating an application, you establish optimization requirements. For example, you might establish the following optimization requirements to address the example issues described above:
     
       
    • Refine the documentation about the IDP and the onboarding and migration   process to include any missing information about the process and its tasks.
      •  
    • Simplify the onboarding and migration process to remove unnecessary tasks, and   automate as many tasks as possible.
      •  
    • Validate that the process accounts for a responsibility assignment that fully   covers the application, the IDP, and the process itself.
      •  
    • Reduce adoption friction by temporarily assigning members of the IDP team to   application teams to act as IDP adoption coaches and consultants.
      •  
    • Refine existing golden paths, or create new ones to cover the application   onboarding and migration.
      •  
    • Reduce adoption friction by implementing a tiered onboarding and migration   process. Each tier would have a different set of requirements according to the   tier. For example, a higher tier would have more stringent requirements than a   lower tier.
      •  
     
    After establishing optimization requirements, you set measurable optimization goals to meet those requirements. For more information about optimization requirements and goals, see Establish your optimization requirements and goals.
    "},{"location":"reference-architectures/accelerating-migrations/#application-onboarding-and-migration-example","title":"Application onboarding and migration example","text":"
    In this section, you explore how the onboarding and migration process looks like for an example. The example that we describe in this section doesn't represent a real production application.
     
    To reduce the scope of the example, we focus the example on the following environments:
     
       
    • Source environment: Amazon Elastic Kubernetes Service (Amazon EKS)
      •  
    • Target environment: GKE
      •  
     
    This document focuses on the onboarding and migration process. For more information about migrating from Amazon EKS to GKE, see Migrate from AWS to Google Cloud: Migrate from Amazon EKS to GKE.
     
    To onboard and migrate the application on the IDP, you follow the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#intake-the-onboarding-and-migration-request-example","title":"Intake the onboarding and migration request (example)","text":"
    In this example, the application onboarding and migration team files a request to onboard and migrate the application on the IDP. To fully present the onboarding and migration process, we assume that IDP cannot find an existing golden path to suggest to onboard and migrate the application, so it forwards the request to the team that manages the IDP for further evaluation.
    "},{"location":"reference-architectures/accelerating-migrations/#plan-application-onboarding-and-migration-example","title":"Plan application onboarding and migration (example)","text":"
    To define timelines and milestones to onboard and migrate the application on the IDP, the application onboarding and migration team prepares a countdown plan:
     Phase Task Countdown [days] Status Assess the application Review the application design -27 Not started Review application dependencies -23 Not started Review CI/CD processes -21 Not started Review data persistence and data management requirements -21 Not started Review FinOps requirements -20 Not started Review compliance requirements -20 Not started Review the application's team practices -19 Not started Assess application refactoring and the IDP -19 Not started Finalize the application onboarding and migration plan -18 Not started Set up the IDP Enhance the IDP N/A Not necessary Configure the IDP -17 Not started Onboard and migrate the application Refactor the application -15 Not started Configure CI/CD workflows -9 Not started Promote from development to staging -6 Not started Perform acceptance testing -5 Not started Migrate data -3 Not started Promote from staging to production -1 Not started Perform the cutover 0 Not started Optimize the application Assess your current environment, teams, and optimization loop 1 Not started Establish your optimization requirements and goals 1 Not started Optimize your environment and your teams 3 Not started Tune the optimization loop 5 Not started 
    To clearly outline responsibility assignments, the application onboarding and migration team defines the following RACI matrix for each phase and task of the process:
     Phase Task Application onboarding and migration team Application development and operations team IDP team Assess the application Review the application design Responsible Accountable Informed Review application dependencies Responsible Accountable Informed Review CI/CD processes Responsible Accountable Informed Review data persistence and data management requirements Responsible Accountable Informed Review FinOps requirements Responsible Accountable Informed Review compliance requirements Responsible Accountable Informed Review the application's team practices Responsible Accountable Informed Assess application refactoring and the IDP Responsible Accountable Consulted Plan application onboarding and migration Responsible Accountable Consulted Set up the IDP Enhance the IDP Accountable Consulted Responsible Configure the IDP Responsible, Accountable Consulted Consulted Onboard and migrate the application Refactor the application Accountable Responsible Consulted Configure CI/CD workflows Responsible, Accountable Consulted Consulted Promote from development to staging Responsible, Accountable Consulted Informed Perform acceptance testing Responsible, Accountable Consulted Informed Migrate data Responsible, Accountable Consulted Consulted Promote from staging to production Responsible, Accountable Consulted Informed Perform the cutover Responsible, Accountable Consulted Informed Optimize the application Assess your current environment, teams, and optimization loop Informed Responsible, Accountable Informed Establish your optimization requirements and goals Informed Responsible, Accountable Informed Optimize your environment and your teams Informed Responsible, Accountable Informed Tune the optimization loop Informed Responsible, Accountable Informed"},{"location":"reference-architectures/accelerating-migrations/#assess-the-application-example","title":"Assess the application (example)","text":"
    In the assessment phase, the application onboarding and migration team assesses the application by completing the assessment phase tasks.
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-application-design-example","title":"Review the application design (example)","text":"
    The application onboarding and migration team reviews the application design, and gathers the following information:
     
       
    1. Application source code. The application source code is available on the    company source code management and hosting solution.
      2.  
    2. Deployable artifacts. The application is fully containerized using a    single Open Container Initiative (OCI) container image. The container image    uses Debian as a base image.
      3.  
    3. Configuration injection. The application supports injecting configuration    using environment variables and configuration files. Environment variables    take precedence over configuration files. The application reads runtime- and    environment-specific configuration from a Kubernetes ConfigMap.
      4.  
    4. Security requirements. Container images need to be scanned for    vulnerabilities. Also, container images need to be verified for authenticity    and bills of materials. The application requires periodic secret rotation.    The application doesn't allow direct access to its production runtime    environment.
      5.  
    5. Identity and access management. The application requires a dedicated    service account with the minimum set of permissions to work correctly.
      6.  
    6. Observability requirements. The application logs messages to stout and    stderr streams, and exposes metrics and tracing in OpenTelemetry format. The    application requires SLO monitoring for uptime and request error rates.
      7.  
    7. Availability and reliability requirements. The application is not    business critical, and can afford two hours of downtime at maximum. The    application is designed to shed load under degraded conditions, and is    capable of automated recovery after a loss of connectivity.
      8.  
    8.  
      Network and connectivity requirements. The application needs:
       
         
      • A /28 IPv4 subnet to account for multiple instances of the application.
        •  
      • A DNS name for each instance of the application.
        •  
      • Connectivity to its data storage systems.
        •  
      • Load balancing across several application instances.
        •  
       
      The application doesn't require any specific service mesh configuration.
       
      9.  
    9.  
      Statefulness. The application stores persistent data on Amazon Relational    Database Service (Amazon RDS) for PostgreSQL and on Amazon Simple Storage    Service (Amazon S3).
       
      10.  
    10. Runtime environment requirements. The application doesn't depend on any     preview Kubernetes features, and doesn't need dependencies outside what is     packaged in its container image.
      11.  
    11. Development tools and environments. The application doesn't have any     dependency on specific IDEs or development hardware.
      12.  
    12. Multi-tenancy requirements. The application doesn't have any     multi-tenancy requirements.
      13.  
    "},{"location":"reference-architectures/accelerating-migrations/#review-application-dependencies-example","title":"Review application dependencies (example)","text":"
    The application onboarding and migration team reviews dependencies on systems that are outside the scope of the application, and gathers the following information:
     
       
    • Internal corporate APIs. The application queries two corporate APIs   through the IDP API gateway.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-cicd-processes-example","title":"Review CI/CD processes (example)","text":"
    The application onboarding and migration team reviews the application's CI/CD processes, and gathers the following information:
     
       
    • A GitHub Action builds deployable artifacts for the application, and stores   artifacts in an Amazon Elastic Container Repository (Amazon ECR).
      •  
    • There is no CD process. To deploy a new version of the application, the   application development and operations team manually runs a scripted workflow   to deploy the application on Amazon EKS.
      •  
    • There is no deployment schedule. The application development and operations   team runs the deployment workflow on demand. In the last two years, the team   deployed the application twice a month, on average.
      •  
    • The deployment process doesn't implement any advanced deployment methodology.
      •  
    • There is no need to migrate deployable artifacts that the CI process built for   previous versions of the application.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-data-persistence-and-data-management-requirements-example","title":"Review data persistence and data management requirements (example)","text":"
    The application onboarding and migration team reviews data persistence and data management requirements, and gathers the following information:
     
       
    • Amazon RDS for PostgreSQL. The application stores and reads data from   three PostgreSQL databases that reside on a single, high-availability Amazon   RDS for PostgreSQL instance. The application uses standard PostgreSQL   features.
      •  
    • Amazon S3. The application stores and reads objects in two Amazon S3   buckets.
      •  
     
    The application onboarding and migration team is also tasked to migrate data from Amazon RDS for PostgreSQL and Amazon S3 to database and object storage services offered by the IDP. In this example, the IDP offers Cloud SQL for PostgreSQL as a database service, and Cloud Storage as an object storage service.
     
    As part of this application dependency review, the application onboarding and migration team assesses the application's Amazon RDS database and the Amazon S3 buckets. For simplicity, we omit details about those assessments from this example. For more information about assessing Amazon RDS and Amazon S3, see the Assess the source environment sections in the following documents:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-finops-requirements-example","title":"Review FinOps requirements (example)","text":"
    The application onboarding and migration team reviews FinOps requirements, and gathers the following information:
     
       
    • The application must not exceed ten thousands USD of maximum aggregated   spending per month.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-compliance-requirements-example","title":"Review compliance requirements (example)","text":"
    The application onboarding and migration team reviews compliance requirements, and gathers the following information:
     
       
    • The application doesn't need to meet any compliance requirements to regulate   data residency and network traffic.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#review-the-applications-team-practices","title":"Review the application's team practices","text":"
    The application onboarding and migration team reviews development and operational practices that the application development and operations team has in place, and gathers the following information:
     
       
    • The team started following an agile development methodology one year ago.
      •  
    • The team is exploring SRE practices, but didn't implement anything in that   regard yet.
      •  
    • The team doesn't have any prior experience with the IDP.
      •  
     
    The application onboarding and migration team suggests the following:
     
       
    • Train the application development and operations team on basic platform   engineering concepts.
      •  
    • Train the team on the architecture of the IDP, how to use the IDP effectively.
      •  
    • Consult with the IDP team to assess potential changes to development and   operational processes after migrating the application on the IDP.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#assess-application-refactoring-and-the-idp-example","title":"Assess application refactoring and the IDP (example)","text":"
    After reviewing the application and its related CI/CD process, the team application onboarding and migration team assesses the refactoring that the application needs to onboard and migrate it on the IDP, scopes the following refactoring tasks:
     
       
    • Support reading objects from objects and writing objects to the IDP's object   storage service. In this example, the IDP offers Cloud Storage as an object   storage service. For more information about refactoring workloads when   migrating from Amazon S3 to Cloud Storage, see the   Migrate data and workloads from Amazon S3 to Cloud Storage section   in Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage.
      •  
    • Update the application configuration to use Cloud SQL for PostgreSQL instead   of Amazon RDS for PostgreSQL.
      •  
    • Support exporting the metrics that the IDP needs to support observability for   the application.
      •  
    • Update the application dependencies to versions that are not impacted by known   vulnerabilities.
      •  
     
    The application onboarding and migration team evaluates the IDP against the application's requirements, and concludes that:
     
       
    • The IDP's current set of services is capable of supporting the application, so   there is no need to extend the IDP to offer additional services.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#finalize-the-application-onboarding-and-migration-plan-example","title":"Finalize the application onboarding and migration plan (example)","text":"
    After completing the application review, the application onboarding and migration team refines the onboarding and migration plan, and validates it in collaboration with technical and non-technical stakeholders.
    "},{"location":"reference-architectures/accelerating-migrations/#set-up-the-idp-example","title":"Set up the IDP (example)","text":"
    After you assess the application and plan the onboarding and migration process, you set up the IDP.
    "},{"location":"reference-architectures/accelerating-migrations/#enhance-the-idp-example","title":"Enhance the IDP (example)","text":"
    The IDP team doesn't need to enhance the IDP to onboard and migrate the application because:
     
       
    • The IDP already offers all the services that the application needs.
      •  
    • The IDP meets the application's compliance and regulatory requirements.
      •  
    • The IDP meets the application's configurability and observability   requirements.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-the-idp-example","title":"Configure the IDP (example)","text":"
    The application onboarding and migration team configures the runtime environments for the application using the IDP: a development environment, a staging environment, and a production environment. For each environment, the application onboarding and migration team completes the following tasks:
     
       
    1.  
      Configures foundational services:
       
         
      1. Creates a new Google Cloud project.
        2.  
      2. Configures IAM roles and service accounts.
        3.  
      3. Configures a VPC and a subnet.
        4.  
      4. Creates DNS records in the DNS zone.
        5.  
       
      2.  
    2.  
      Provisions and configures a GKE cluster for the application.
       
      3.  
    3. Provisions and configures a Cloud SQL for PostgreSQL instance.
      4.  
    4. Provisions and configures two Cloud Storage buckets.
      5.  
    5. Provisions and configures an Artifact Registry repository for container    images.
      6.  
    6. Instruments Cloud Operations Suite to observe the application.
      7.  
    7. Configures Cloud Billing budget and budget alerts for the application.
      8.  
    "},{"location":"reference-architectures/accelerating-migrations/#onboard-and-migrate-the-application-example","title":"Onboard and migrate the application (example)","text":"
    To onboard and migrate the application, the application development and operations team refactors the application and then the application onboarding and migration team proceeds with the onboarding and migration process.
    "},{"location":"reference-architectures/accelerating-migrations/#refactor-the-application-example","title":"Refactor the application (example)","text":"
    The application development and operations team refactors the application as follows:
     
       
    1. Refactors the application to read from and write objects to Cloud Storage,    instead of Amazon S3.
      2.  
    2. Updates the application configuration to use the Cloud SQL for PostgreSQL,    instance instead of the Amazon RDS for PostgreSQL instance.
      3.  
    3. Exposes the metrics that the IDP needs to observe the application.
      4.  
    4. Update application dependencies that are affected by known vulnerabilities.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#configure-cicd-workflows-example","title":"Configure CI/CD workflows (example)","text":"
    To configure CI/CD workflows, the application onboarding and migration team does the following:
     
       
    1. Refactors the application CI workflow to push container images to the    Artifact Registry repository, in addition to Amazon ECR.
      2.  
    2. Implements a Cloud Deploy pipeline to automatically deploy the application,    and promote it across runtime environments.
      3.  
    3. Deploys the application in the development environment using the Cloud Deploy    pipeline.
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-the-application-from-development-to-staging","title":"Promote the application from development to staging","text":"
    After deploying the application in the development environment, the application onboarding and migration team:
     
       
    1. Tests the application, and verifies that it works as expected.
      2.  
    2. Promotes the application from the development environment to the staging    environment.
      3.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-acceptance-testing-example","title":"Perform acceptance testing (example)","text":"
    After promoting the application from the development environment to the staging environment, the application onboarding and migration team performs acceptance testing.
     
    To perform acceptance testing to validate the application's real-world user journeys and business processes, the application onboarding and migration team consults with the application development and operations team.
     
    The application onboarding and migration team performs acceptance testing as follows:
     
       
    1. Ensures that the application works as expected when dealing with amounts of    data and traffic that are similar to production ones.
      2.  
    2.  
      Validates that the application works as designed under degraded conditions,    and that it recovers once the issues are resolved. The application onboarding    and migration team tests the following scenarios:
       
         
      • Loss of connectivity to the database
        •  
      • Loss of connectivity to the object storage
        •  
      • Degradation of the CI/CD pipeline that blocks deployments
        •  
      • Tentative exploitation of short-lived credentials, such as access tokens
        •  
      • Excessive application load
        •  
       
      3.  
    3.  
      Verifies that observability and alerting for the application are correctly    configured.
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#migrate-data-example","title":"Migrate data (example)","text":"
    After completing acceptance testing for the application, the application onboarding and migration team migrates data from the source environment to the Google Cloud environment as follows:
     
       
    1. Migrate data from Amazon RDS for PostgreSQL to Cloud SQL for PostgreSQL.
      2.  
    2. Migrate data from Amazon S3 to Cloud Storage.
      3.  
     
    For simplicity, this document doesn't describe the details of migrating from Amazon RDS and Amazon S3 to Google Cloud. For more information about migrating from Amazon RDS and Amazon S3 to Google Cloud, see:
     
       
    • Migrate from AWS to Google Cloud: Migrate from Amazon S3 to Cloud Storage
      •  
    • Migrate from AWS to Google Cloud: Migrate from Amazon RDS and Amazon Aurora for PostgreSQL to Cloud SQL and AlloyDB for PostgreSQL
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#promote-from-staging-to-production-example","title":"Promote from staging to production (example)","text":"
    After performing acceptance testing and after migrating data to the Google Cloud environment, the application onboarding and migration team:
     
       
    1. Promotes the application from the staging environment to the production    environment using the Cloud Deploy pipeline.
      2.  
    2.  
      Ensures the application's operational readiness by verifying that the    application:
       
      3.  
    3.  
      Correctly connects to the Cloud SQL for PostgreSQL instance
       
         
      • Has access to the Cloud Storage buckets
        •  
      • Exposes endpoints through Cloud Load Balancing
        •  
       
      4.  
    "},{"location":"reference-architectures/accelerating-migrations/#perform-the-cutover-example","title":"Perform the cutover (example)","text":"
    After promoting the application to the production environment, and ensuring that the application is operationally ready, the application onboarding and migration team:
     
       
    1. Configures the production environment to gradually route requests to the    application in 5% increments, until all the requests are routed to the Google    Cloud environment.
      2.  
    2. Refactors the CI workflow to push container images to Artifact Registry only.
      3.  
    3. Takes backups to ensure that a rollback is possible, in case of unanticipated    issues.
      4.  
    4. Retires the application in the source environment.
      5.  
    "},{"location":"reference-architectures/accelerating-migrations/#optimize-the-application-example","title":"Optimize the application (example)","text":"
    After performing the cutover, the application development and operations team takes over the maintenance of the application, and establishes the following optimization requirements:
     
       
    • Refine the CD process by adopting a canary deployment methodology.
      •  
    •  
      Reduce the application's operational costs by:
       
         
      • Tuning the configuration of the GKE cluster
        •  
      • Enabling the Cloud Storage Autoclass feature
        •  
       
      •  
     
    After establishing optimization requirements, the application development and operations team completes the rest of the tasks of the optimization phase.
    "},{"location":"reference-architectures/accelerating-migrations/#whats-next","title":"What's next","text":"
       
    • Learn how to   Migrate from Amazon EKS to Google Kubernetes Engine (GKE).
      •  
    • Learn when to   find help for your migrations.
      •  
    "},{"location":"reference-architectures/accelerating-migrations/#contributors","title":"Contributors","text":"
    Authors:
     
       
    • Marco Ferrari | Cloud Solutions   Architect
      •  
    • Paul Revello | Cloud Solutions   Architect
      •  
     
    Other contributors:
     
       
    • Ben Good | Solutions Architect
      •  
    • Shobhit Gupta |   Solutions Architect
      •  
    • James Brookbank | Solutions Architect
      •  
    "},{"location":"reference-architectures/automated-password-rotation/","title":"Overview","text":"
    Secrets rotation is a broadly accepted best practice across the information technology industry. However, often times it is cumbersome and disruptive process. In this guide you will use Google Cloud tools to automate the process of rotating passwords for a Cloud SQL instance. This method could easily be extended to other tools and types of secrets.
    "},{"location":"reference-architectures/automated-password-rotation/#storing-passwords-in-google-cloud","title":"Storing passwords in Google Cloud","text":"
    In Google Cloud, secrets including passwords can be stored using many different tools including common open source tools such as Vault, however in this guide, you will use Secret Manager, Google Cloud's fully managed product for securely storing secrets. Regardless of the tool you use, passwords stored should be further secured. When using Secret Manager, following are some of the ways you can further secure your secrets:
     
       
    1.  
      Limiting access : The secrets should be readable writable only through     the Service Accounts via IAM roles. The principle     of least privilege must be followed while granting roles to the service     accounts.
       
      2.  
    2.  
      Encryption : The secrets should be encrypted. Secret     Manager encrypts the secret at rest using AES-256 by     default. But you can use your own encryption keys, customer-managed     encryption keys (CMEK) to encrypt your secret at rest. For details, see     Enable customer-managed encryption keys for Secret     Manager.
       
      3.  
    3.  
      Password rotation : The passwords stored in the secret manager should be     rotated on a regular basis to reduce the risk of a security incident.
       
      4.  
    "},{"location":"reference-architectures/automated-password-rotation/#why-password-rotation","title":"Why password rotation","text":"
    Security best practices require us to regularly rotate the passwords in our stack. Changing the password mitigates the risk in the event where passwords are compromised.
    "},{"location":"reference-architectures/automated-password-rotation/#how-to-rotate-passwords","title":"How to rotate passwords","text":"
    Manually rotating the passwords is an antipattern and should not be done as it exposes the password to the human rotating it and may result in security and system incidents. Manual rotation processes also introduce the risk that the rotation isn't actually performed due to human error, for example forgetting or typos.
     
    This necessitates having a workflow that automates password rotation. The password could be of an application, a database, a third-party service or a SaaS vendor etc.
    "},{"location":"reference-architectures/automated-password-rotation/#automatic-password-rotation","title":"Automatic password rotation","text":"
    Typically, rotating a password requires these steps:
     
       
    • Change the password in the underlying software or system
      •  
     
    (such as applications,databases, SaaS).
     
       
    •  
      Update Secret Manager to store the new password.
       
      •  
    •  
      Restart the applications that use that password. This will make the
       
      •  
     
    application source the latest passwords.
     
    The following architecture represents a general design for a systems that can rotate password for any underlying software/system.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow","title":"Workflow","text":"
       
    • A pipeline or a cloud scheduler job sends a message to a   pub/sub topic. The message contains the information about the password that is   to be rotated. For example, this information may include secret ID in secret   manager, database instance and username if it is a database password.
      •  
    • The message arriving to the pub/sub topic triggers a Cloud Run   Function that reads the message and gathers information as   supplied in the message.
      •  
    • The function changes the password in the corresponding system. For example, if   the message contained a database instance, database name and user,the function   changes the password for that user in the given database.
      •  
    • The function updates the password in secret manager to reflect the new   password. It knows what secret ID to update since it was provided in the   pub/sub message.
      •  
    • The function publishes a message to a different pub/sub topic indicating that   the password has been rotated. This topic can be subscribed any application or   system that may want to know in the event of password rotation, whether to   re-start themselves or perform any other task.
      •  
    "},{"location":"reference-architectures/automated-password-rotation/#example-deployment-for-automatic-password-rotation-in-cloudsql","title":"Example deployment for automatic password rotation in CloudSQL","text":"
    The following architecture demonstrates a way to automatically rotate CloudSQL password.
     
    "},{"location":"reference-architectures/automated-password-rotation/#workflow-of-the-example-deployment","title":"Workflow of the example deployment","text":"
       
    • A Cloud Scheduler job is scheduled to run every 1st day on   the month. The jobs publishes a message to a Pub/Sub topic containing secret   ID, Cloud SQL instance name, database, region and database user in the   payload.
      •  
    • The message arrival on the pub/sub topic triggers a Cloud Run   Function, which uses the information provided in the message   to connect to the CloudSQL instance via Serverless VPC   Connector and changes the password. The function uses a   service account that has IAM roles required to   connect to the Cloud Sql instance.
      •  
    • The function then updates the secret in Secret Manager.
      •  
     
    Note : The architecture doesn't show the flow to restart the application after the password rotation as shown in thee Generic architecture but it can be added easily with minimal changes to the Terraform code.
    "},{"location":"reference-architectures/automated-password-rotation/#deploy-the-architecture","title":"Deploy the architecture","text":"
    The code to build the architecture has been provided with this repository. Follow these instructions to create the architecture and use it:
     
       
    1.  
      Open Cloud Shell on Google Cloud Console and log in with your     credentials.
       
      2.  
    2.  
      If you want to use an existing project, get role/project.owner role on the     project and set the environment in Cloud Shell as shown below. Then, move to     step 4.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the existing project.
       
      3.  
    3.  
      If you want to create a new GCP project run the following commands in Cloud     Shell.
       
       #set shell environment variable\n export PROJECT_ID=<PROJECT_ID>\n #create project\n gcloud projects create ${PROJECT_ID} --folder=<FOLDER_ID>\n #associate the project with billing account\n gcloud billing projects link ${PROJECT_ID} --billing-account=<BILLING_ACCOUNT_ID>\n
       
      Replace <PROJECT_ID> with the ID of the new project. Replace <BILLING_ACCOUNT_ID> with the billing account ID that the project should be associated with.
       
      4.  
    4.  
      Set the project ID in Cloud Shell and enable APIs in the project:
       
       gcloud config set project ${PROJECT_ID}\n gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project ${PROJECT_ID}\n
       
      5.  
    5.  
      Download the Git repository containing the code to build the example     architecture:
       
       cd ~\n git clone https://github.com/GoogleCloudPlatform/platform-engineering\n cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n terraform init\n terraform plan -var \"project_id=$PROJECT_ID\"\n terraform apply -var \"project_id=$PROJECT_ID\" --auto-approve\n
       
      Note: It takes around 30 mins for the entire architecture to get deployed.
       
      6.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-the-deployed-architecture","title":"Review the deployed architecture","text":"
    Once the Terraform apply has successfully finished, the example architecture will be deployed in the your Google Cloud project. Before exercising the rotation process, review and verify the deployment in the Google Cloud Console.
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-sql-database","title":"Review Cloud SQL database","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL.     Confirm that cloudsql-for-pg is present in the instance list.
      2.  
    2. Click on cloudsql-for-pg, to open the instance details page.
      3.  
    3. In the left hand menu select Users. Confirm you see a user with the name     user1.
      4.  
    4. In the left hand menu select Databases. Confirm you see see a database     named test.
      5.  
    5. In the left hand menu select Overview.
      6.  
    6. In the Connect to this instance section, note that only     Private IP address is present and no public IP address. This restricts     access to the instance over public network.
      7.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-secret-manager","title":"Review Secret Manager","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value, you will use this in the next section to confirm     access to the Cloud SQL instance.
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-scheduler-job","title":"Review Cloud Scheduler job","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler. Confirm that     password-rotator-job is present in the Scheduler Jobs list.
      2.  
    2. Click on password-rotator-job, confirm it is configured to run on 1st of     every month.
      3.  
    3.  
      Click Continue to see execution configuration. Confirm the following     settings:
       
         
      • Target type is Pub/Sub
        •  
      • Select a Cloud Pub/Sub topic is set to pswd-rotation-topic
        •  
      • Message body contains a JSON object with the details of the Cloud SQL   isntance and secret to be rotated.
        •  
       
      4.  
    4.  
      Click Cancel, to exit the Cloud Scheduler job details.
       
      5.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-pubsub-topic-configuration","title":"Review Pub/Sub topic configuration","text":"
       
    1. In the Cloud Console, using the naviagion menu select Analytics > Pub/Sub.
      2.  
    2. In the left hand menu select Topic. Confirm that pswd-rotation-topic is     present in the topics list.
      3.  
    3. Click on pswd-rotation-topic.
      4.  
    4. In the Subscriptions tab, click on Subscription ID for the rotator Cloud     Function.
      5.  
    5. Click on the Details tab. Confirm, the Audience tag shows the rotator     Cloud Function.
      6.  
    6. In the left hand menu select Topic.
      7.  
    7. Click on pswd-rotation-topic.
      8.  
    8. Click on the Details tab.
      9.  
    9. Click on the schema in the Schema name field.
      10.  
    10. In the Details, confirm that the schema contains these keys: secretid,     instance_name, db_user, db_name and db_location. These keys will be     used to identify what database and user password is to be rotated.
      11.  
    "},{"location":"reference-architectures/automated-password-rotation/#review-cloud-run-function","title":"Review Cloud Run Function","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions. Confirm that pswd_rotator_function is     present in the list.
      2.  
    2. Click on pswd_rotator_function.
      3.  
    3. Click on the Trigger tab. Confirm that the field Receive events from has     the Pub/Sub topic pswd-rotation-topic. This indicates that the function     will run when a message arrives to that topic.
      4.  
    4. Click on the Details tab. Confirm that under Network Settings VPC     connector is set to connector-for-sql. This allows the function to connect     to the CloudSQL over private IPs.
      5.  
    5. Click on the Source tab to see the python code that the function executes.
      6.  
     
    Note: For the purpose of this tutorial, the secret is accessible to the human users and not encrypted. See the section and Secret Manager best practice
    "},{"location":"reference-architectures/automated-password-rotation/#verify-that-you-are-able-to-connect-to-the-cloud-sql-instance","title":"Verify that you are able to connect to the Cloud SQL instance","text":"
       
    1. In the Cloud Console, using the naviagion menu select Databases > SQL
      2.  
    2. Click on cloudsql-for-pg
      3.  
    3. In the left hand menu select Cloud SQL Studio.
      4.  
    4. In Database dropdown, choose test.
      5.  
    5. In User dropdown, choose user1.
      6.  
    6. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      7.  
    7. Click Authenticate. Confirm you were able to log in to the database.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#rotate-the-cloud-sql-password","title":"Rotate the Cloud SQL password","text":"
    Typically, the Cloud Scheduler will automatically run on 1st day of every month triggering password rotation. However, for this tutorial you will run the Cloud Scheduler job manually, which causes the Cloud Run Function to generate a new password, update it in Cloud SQL and store it in Secret Manager.
     
       
    1. In the Cloud Console, using the naviagion menu select     Integration Services > Cloud Scheduler.
      2.  
    2. For the scheduler job password-rotator-job. Click the three dots icon and     select Force run.
      3.  
    3. Verify that the Status of last execution shows Success.
      4.  
    4. In the Cloud Console, using the naviagion menu select     Serverless > Cloud Run Functions.
      5.  
    5. Click function named pswd_rotator_function.
      6.  
    6. Select the Logs tab.
      7.  
    7. Review the logs and verify the function has run and completed without     errors. Successful completion will be noted with log entries containing     Secret cloudsql-pswd changed in Secret Manager!,     DB password changed successfully! and     DB password verified successfully!.
      8.  
    "},{"location":"reference-architectures/automated-password-rotation/#test-the-new-password","title":"Test the new password","text":"
       
    1. In the Cloud Console, using the naviagion menu select     Security > Secret Manager. Confirm that cloudsql-pswd is present in the     list.
      2.  
    2. Click on cloudsql-pswd. Note you should now see a new version, version 2     of the secret.
      3.  
    3. Click three dots icon and select View secret value to view the password     for Cloud SQL database.
      4.  
    4. Copy the secret value.
      5.  
    5. In the Cloud Console, using the naviagion menu select Databases > SQL
      6.  
    6. Click on cloudsql-for-pg
      7.  
    7. In the left hand menu select Cloud SQL Studio.
      8.  
    8. In Database dropdown, choose test.
      9.  
    9. In User dropdown, choose user1.
      10.  
    10. In Password textbox paste the password copied from the cloudsql-pswd     secret.
      11.  
    11. Click Authenticate. Confirm you were able to log in to the database.
      12.  
    "},{"location":"reference-architectures/automated-password-rotation/#destroy-the-architecture","title":"Destroy the architecture","text":"
      cd platform-engineering/reference-architectures/automated-password-rotation/terraform\n\n  terraform init\n  terraform plan -var \"project_id=$PROJECT_ID\"\n  terraform destroy -var \"project_id=$PROJECT_ID\" --auto-approve\n
    "},{"location":"reference-architectures/automated-password-rotation/#conclusion","title":"Conclusion","text":"
    In this tutorial, you saw a way to automate password rotation on Google Cloud. First, you saw a generic reference architecture that can be used to automate password rotation in any password management system. In the later section, you saw an example deployment that uses Google Cloud services to rotate password of Cloud Sql database in Google Cloud Secret Manager.
     
    Implementing an automatic flow to rotate passwords takes away manual overhead and provide seamless way to tighten your password security. It is recommended to create an automation flow that runs on a regular schedule but can also be easily triggered manually when needed. There can be many variations of this architecture that can be adopted. For example, you can directly trigger a Cloud Run Function from a Google Cloud Scheduler job without sending a message to pub/sub if you don't want to broadcast the password rotation. You should identify a flow that fits your organization requirements and modify the reference architecture to implement it.
    "},{"location":"reference-architectures/backstage/","title":"Backstage on Google Cloud","text":"
    A collection of resources related to utilizing Backstage on Google Cloud.
    "},{"location":"reference-architectures/backstage/#backstage-plugins-for-google-cloud","title":"Backstage Plugins for Google Cloud","text":"
    A repository for various plugins can be found here -> google-cloud-backstage-plugins
    "},{"location":"reference-architectures/backstage/#backstage-quickstart","title":"Backstage Quickstart","text":"
    This is an example deployment of Backstage on Google Cloud with various Google Cloud services providing the infrastructure.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/","title":"Backstage on Google Cloud Quickstart","text":"
    This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts related to hosting Backstage on Google Cloud.
     
    NOTE: This environment is not intended to be a long lived environment. It is intended for temporary demonstration and learning purposes. You will need to modify the configurations provided to align with your orginazations needs. Along the way the guide will make callouts to tasks or areas that should be productionized in for long lived deployments.
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#architecture","title":"Architecture","text":"
    The following diagram depicts the high level architecture of the infrastucture that will be deployed.
     
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#requirements-and-assumptions","title":"Requirements and Assumptions","text":"
    To keep this guide simple it makes a few assumptions. Where the are alternatives we have linked to some additional documentation.
     
       
    1. The Backstage quick start will be deployed in a new project that you will     manually create. If you want to use a project managed through Terraform     refer to the Terraform managed project     section.
      2.  
    2. Identity Aware Proxy (IAP) will be used for controlling access to     Backstage.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2. Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-creation","title":"Project Creation","text":"
    In this section you prepare your project for deployment.
     
       
    1.  
      Go to the project selector page in the Cloud Console.     Select or create a Cloud project.
       
      2.  
    2.  
      Make sure that billing is enabled for your Google Cloud project. Learn how     to confirm billing is enabled for your project.
       
      3.  
    3.  
      In Cloud Shell, set environment variables with the ID of your project:
       
      export PROJECT_ID=<INSERT_YOUR_PROJECT_ID>\ngcloud config set project \"${PROJECT_ID}\"\n
       
      4.  
    4.  
      Clone the repository and change directory to the guide directory
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering && \\\ncd platform-engineering/reference-architectures/backstage/backstage-quickstart\n
       
      5.  
    5.  
      Set environment variables
       
      export BACKSTAGE_QS_BASE_DIR=$(pwd) && \\\nsed -n -i -e '/^export BACKSTAGE_QS_BASE_DIR=/!p' -i -e '$aexport  \\\nBACKSTAGE_QS_BASE_DIR=\"'\"${BACKSTAGE_QS_BASE_DIR}\"'\"' ${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#project-configuration","title":"Project Configuration","text":"
       
    1.  
      Set the project environment variables in Cloud Shell
       
      export BACKSTAGE_QS_STATE_BUCKET=\"${PROJECT_ID}-terraform\"\nexport IAP_USER_DOMAIN=\"<your org's domain>\"\nexport IAP_SUPPORT_EMAIL=\"<your org's support email>\"\n
       
      2.  
    2.  
      Create a Cloud Storage bucket to store the Terraform state
       
      gcloud storage buckets create gs://${BACKSTAGE_QS_STATE_BUCKET} --project ${PROJECT_ID}\n
       
      3.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#deploy-backstage","title":"Deploy Backstage","text":"
    Before running Terraform, make sure that the Service Usage API and Service Management API are enabled.
     
       
    1.  
      Enable Service Usage API and Service Management API
       
      gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iap.googleapis.com \\\n  serviceusage.googleapis.com \\\n  servicemanagement.googleapis.com\n
       
      2.  
    2.  
      Setup the Identity Aware Proxy brand
       
      gcloud iap oauth-brands create \\\n  --application_title=\"IAP Secured Backstage\" \\\n  --project=\"${PROJECT_ID}\" \\\n  --support_email=\"${IAP_SUPPORT_EMAIL}\"\n
       
      Capture the brand name in an environment variable, it will be in the format of: projects/[your_project_number]/brands/[your_project_number].
       
      export IAP_BRAND=<your_brand_name>\n
       
      3.  
    3.  
      Using the brand name create the IAP client.
       
      gcloud iap oauth-clients create \\\n  ${IAP_BRAND} \\\n  --display_name=\"IAP Secured Backstage\"\n
       
      Capture the client_id and client_secret in environment variables. For the client_id we only need the last value of the string, it will be in the format of: 549085115274-ksi3n9n41tp1vif79dda5ofauk0ebes9.apps.googleusercontent.com
       
      export IAP_CLIENT_ID=\"<your_client_id>\"\nexport IAP_SECRET=\"<your_iap_secret>\"\n
       
      4.  
    4.  
      Set the configuration variables
       
      sed -i \"s/YOUR_STATE_BUCKET/${BACKSTAGE_QS_STATE_BUCKET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backend.tf\nsed -i \"s/YOUR_PROJECT_ID/${PROJECT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_USER_DOMAIN/${IAP_USER_DOMAIN}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SUPPORT_EMAIL/${IAP_SUPPORT_EMAIL}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_CLIENT_ID/${IAP_CLIENT_ID}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\nsed -i \"s/YOUR_IAP_SECRET/${IAP_SECRET}/g\" ${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Create the resources
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan\n
       
      Initial run of the Terraform may result in errors due to they way the API services are asyrchonously enabled. Re-running the terraform usually resolves the errors.
       
      This will take a while to create all of the required resources, figure somewhere between 15 and 20 minutes.
       
      6.  
    6.  
      Build the container image for Backstage
       
      cd manifests/cloudbuild\ngcloud builds submit .\n
       
      The output of that command will include a fully qualified image path similar to: us-central1-docker.pkg.dev/[your_project]/backstage-qs/backstage-quickstart:d747db2a-deef-4783-8a0e-3b36e568f6fc Using that value create a new environment variable.
       
      export IMAGE_PATH=\"<your_image_path>\"\n
       
      This will take approximately 10 minutes to build and push the image.
       
      7.  
    7.  
      Configure Cloud SQL postgres user for password authentication.
       
      gcloud sql users set-password postgres --instance=backstage-qs --prompt-for-password\n
       
      8.  
    8.  
      Grant the backstage workload service account create database permissions.
       
      a. In the Cloud Console, navigate to SQL
       
      b. Select the database instance
       
      c. In the left menu select Cloud SQL Studio
       
      d. Choose the postgres database and login with the postgres user and password you created in step 4.
       
      e. Run the following sql commands, to grant create database permissions
       
      ALTER USER \"backstage-qs-workload@[your_project_id].iam\" CREATEDB\n
       
      9.  
    9.  
      Perform an initial deployment of Kubernetes resources.
       
      cd ../k8s\nsed -i \"s%CONTAINER_IMAGE%${IMAGE_PATH}%g\" deployment.yaml\ngcloud container clusters get-credentials backstage-qs --region us-central1 --dns-endpoint\nkubectl apply -f .\n
       
      10.  
    10.  
      Capture the IAP audience, the Backend Service may take a few minutes to     appear.
       
      a. In the Cloud Console, navigate to Security > Identity-Aware Proxy
       
      b. Verify the IAP option is set to enabled. If not enable it now.
       
      b. Choose Get JWT audience code from the three dot menu on the right side of your Backend Service.
       
      c. The value will be in the format of: /projects/<your_project_number>/global/backendServices/<numeric_id>. Using that value create a new environment variable.
       
      export IAP_AUDIENCE_VALUE=\"<your_iap_audience_value>\"\n
       
      11.  
    11.  
      Redeploy the Kubernetes manifests with the IAP audience
       
      sed -i \"s%IAP_AUDIENCE_VALUE%${IAP_AUDIENCE_VALUE}%g\" deployment.yaml\nkubectl apply -f .\n
       
      12.  
    12.  
      In a browser navigate to you backstage endpoint. The URL will be similar to     https://qs.endpoints.[your_project_id].cloud.goog
       
      13.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleanup","title":"Cleanup","text":"
       
    1.  
      Destroy the resources using Terraform destroy
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nterraform init && \\\nterraform destroy -auto-approve && \\\nrm -rf .terraform .terraform.lock.hcl\n
       
      2.  
    2.  
      Delete the project
       
      gcloud projects delete ${PROJECT_ID}\n
       
      3.  
    3.  
      Remove Terraform files and temporary files
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\nrm -rf \\\n.terraform \\\n.terraform.lock.hcl \\\ninitialize/.terraform \\\ninitialize/.terraform.lock.hcl \\\ninitialize/backend.tf.local \\\ninitialize/state\n
       
      4.  
    4.  
      Reset the TF variables file
       
      cd ${BACKSTAGE_QS_BASE_DIR} && \\\ncp backstage-qs-auto.tfvars.local backstage-qs.auto.tfvars\n
       
      5.  
    5.  
      Remove the environment variables
       
      sed \\\n-i -e '/^export BACKSTAGE_QS_BASE_DIR=/d' \\\n${HOME}/.bashrc\n
       
      6.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#advanced-options","title":"Advanced Options","text":""},{"location":"reference-architectures/backstage/backstage-quickstart/#terraform-managed-project","title":"Terraform managed project","text":"
    In some instances you will need to create and manage the project through Terraform. This quickstart provides a sample process and Terraform to create and destory the project via Terraform.
     
    To run this part of the quick start you will need the following information and permissions.
     
       
    • Billing account ID
      •  
    • Organization or folder ID
      •  
    • roles/billing.user IAM permissions on the billing account specified
      •  
    • roles/resourcemanager.projectCreator IAM permissions on the organization or   folder specified
      •  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#creating-a-terraform-managed-project","title":"Creating a Terraform managed project","text":"
       
    1.  
      Set the configuration variables
       
      nano ${BACKSTAGE_QS_BASE_DIR}/initialize/initialize.auto.tfvars\n
       
      environment_name  = \"qs\"\niapUserDomain = \"\"\niapSupportEmail = \"\"\nproject = {\n  billing_account_id = \"XXXXXX-XXXXXX-XXXXXX\"\n  folder_id          = \"############\"\n  name               = \"backstage\"\n  org_id             = \"############\"\n}\n
       
      Values required :
       
         
      • environment_name: the name of the environment (defaults to qs for   quickstart)
        •  
      • iapUserDomain: the root domain of the GCP Org that the Backstage users   will be in
        •  
      • iapSupportEmail: support contact for the IAP brand
        •  
      • project.billing_account_id: the billing account ID
        •  
      • project.name: the prefix for the display name of the project, the full   name will be <project.name>-<environment_name>
        •  
      • Either project.folder_id OR project.org_id
           
        • project.folder_id: the Google Cloud folder ID
          •  
        • project.org_id: the Google Cloud organization ID
          •  
         
        •  
       
      2.  
    2.  
      Authorize gcloud
       
      gcloud auth login --activate --no-launch-browser --quiet --update-adc\n
       
      3.  
    3.  
      Create a new project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize\nterraform init && \\\nterraform plan -input=false -out=tfplan && \\\nterraform apply -input=false tfplan && \\\nrm tfplan && \\\nterraform init -force-copy -migrate-state && \\\nrm -rf state\n
       
      4.  
    4.  
      Set the project environment variables in Cloud Shell
       
      PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars |\nawk -F\"=\" '{print $2}' | xargs)\n
       
      5.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#cleaning-up-a-terraform-managed-project","title":"Cleaning up a Terraform managed project","text":"
       
    1.  
      Destroy the project
       
      cd ${BACKSTAGE_QS_BASE_DIR}/initialize && \\\nTERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F\"=\" '{print $2}' |\nxargs) && \\\ncp backend.tf.local backend.tf && \\\nterraform init -force-copy -lock=false -migrate-state && \\\ngsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \\\nterraform init && \\\nterraform destroy -auto-approve  && \\\nrm -rf .terraform .terraform.lock.hcl state/\n
       
      2.  
    "},{"location":"reference-architectures/backstage/backstage-quickstart/#re-using-an-existing-project","title":"Re-using an Existing Project","text":"
    In situations where you have run this quickstart before and then cleaned-up the resources but are re-using the project, it might be neccasary to restore the endpoints from a deleted state first.
     
    BACKSTAGE_QS_PREFIX=$(grep environment_name \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\nBACKSTAGE_QS_PROJECT_ID=$(grep environment_project_id \\\n${BACKSTAGE_QS_BASE_DIR}/backstage-qs.auto.tfvars | awk -F\"=\" '{print $2}' | xargs)\ngcloud endpoints services undelete \\\n${BACKSTAGE_QS_PREFIX}.endpoints.${BACKSTAGE_QS_PROJECT_ID}.cloud.goog \\\n--quiet 2>/dev/null\n
    "},{"location":"reference-architectures/cloud_deploy_flow/","title":"Platform Engineering Deployment Demo","text":""},{"location":"reference-architectures/cloud_deploy_flow/#background","title":"Background","text":"
    Platform engineering focuses on providing a robust framework for managing the deployment of applications across various environments. One of the critical components in this field is the automation of application deployments, which streamlines the entire process from development to production.
     
    Most organizations have predefined rules around security, privacy, deployment, and change management to ensure consistency and compliance across environments. These rules often include automated security scans, privacy checks, and controlled release protocols that track all changes in both production and pre-production environments.
     
    In this demo, the architecture is designed to show how a deployment tool like Cloud Deploy can integrate smoothly into such workflows, supporting both automation and oversight. The process starts with release validation, ensuring that only compliant builds reach the release stage. Rollout approvals then offer flexibility, allowing teams to implement either manual checks or automated responses depending on specific requirements.
     
    This setup provides a blueprint for organizations to streamline deployment cycles while maintaining robust governance. By using this demo, you can see how these components work together, from container build through deployment, in a way that minimizes disruption to existing processes and aligns with typical organizational change management practices.
     
    This demo showcases a complete workflow that begins with the build of a container and progresses through various stages, ultimately resulting in the deployment of a new application.
    "},{"location":"reference-architectures/cloud_deploy_flow/#overview-of-the-demo","title":"Overview of the Demo","text":"
    This demo illustrates the end-to-end deployment process, starting from the container build phase. Here's a high-level overview of the workflow:
     
       
    1.  
      Container Build Process: The demo begins when a container is built-in     Cloud Build. Upon completion, a notification is sent to a Pub/Sub message     queue.
       
      2.  
    2.  
      Release Logic: A Cloud Run Function subscribes to this message queue,     assessing whether a release should be created. If a release is warranted, a     message is sent to a \"Command Queue\" (another Pub/Sub topic).
       
      3.  
    3.  
      Creating a Release: A dedicated function listens to the \"Command Queue\"     and communicates with Cloud Deploy to create a new release. Once the release     is created, a notification is dispatched to the Pub/Sub Operations topic.
       
      4.  
    4.  
      Rollout Process: Another Cloud Function picks up this notification and     initiates the rollout process by sending a createRolloutRequest to the     \"Command Queue.\"
       
      5.  
    5.  
      Approval Process: Since rollouts typically require approval, a     notification is sent to the cloud-deploy-approvals Pub/Sub queue. An     approval function then picks up this message, allowing you to implement your     custom logic or utilize the provided site Demo to return JSON, such as     { \"manualApproval\": \"true\" }.
       
      6.  
    6.  
      Deployment: Once approved, the rollout proceeds, and the new application     is deployed.
       
      7.  
     
    "},{"location":"reference-architectures/cloud_deploy_flow/#prerequisites","title":"Prerequisites","text":"
       
    • A GCP project with billing enabled
      •  
    • The following APIs must be enabled in your GCP project:
         
      • compute.googleapis.com
        •  
      • iam.googleapis.com
        •  
      • cloudresourcemanager.googleapis.com
        •  
       
      •  
    • Ensure you have the necessary IAM roles to manage these resources.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#iam-roles-used-by-terraform","title":"IAM Roles used by Terraform","text":"
    To run this demo, the following IAM roles will be granted to the service account created by the Terraform configuration:
     
       
    • roles/iam.serviceAccountUser: Allows management of service accounts.
      •  
    • roles/logging.logWriter: Grants permission to write logs.
      •  
    • roles/artifactregistry.writer: Enables writing to Artifact Registry.
      •  
    • roles/storage.objectUser: Provides access to Cloud Storage objects.
      •  
    • roles/clouddeploy.jobRunner: Allows execution of Cloud Deploy jobs.
      •  
    • roles/clouddeploy.releaser: Grants permissions to release configurations in   Cloud Deploy.
      •  
    • roles/run.developer: Enables deploying and managing Cloud Run services.
      •  
    • roles/cloudbuild.builds.builder: Allows triggering and managing Cloud Build   processes.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#gcp-services-enabled-by-terraform","title":"GCP Services enabled by Terraform","text":"
    The following Google Cloud services must be enabled in your project to run this demo:
     
       
    • pubsub.googleapis.com: Enables Pub/Sub for messaging between services.
      •  
    • clouddeploy.googleapis.com: Allows use of Cloud Deploy for managing   deployments.
      •  
    • cloudbuild.googleapis.com: Enables Cloud Build for building and deploying   applications.
      •  
    • compute.googleapis.com: Provides access to Compute Engine resources.
      •  
    • cloudresourcemanager.googleapis.com: Allows management of project-level   permissions and resources.
      •  
    • run.googleapis.com: Enables Cloud Run for deploying and running   containerized applications.
      •  
    • cloudfunctions.googleapis.com: Allows use of Cloud Functions for   event-driven functions.
      •  
    • eventarc.googleapis.com: Enables Eventarc for routing events from sources to   targets.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/#getting-started","title":"Getting Started","text":"
    To run this demo, follow these steps:
     
       
    1.  
      Fork and Clone the Repository: Start by forking this repository to your     GitHub account (So you can connect GCP to this repository), then clone it to     your local environment. After cloning, change your directory to the     deployment demo:
       
      cd platform-engineering/reference-architectures/cloud_deploy_flow\n
       
      2.  
    2.  
      Set Up Environment Variables or Variables File: You can set the     necessary variables either by exporting them as environment variables or by     creating a terraform.tfvars file. Refer to variables.tf for more details     on each variable. Ensure the values match your Google Cloud project and     GitHub configuration.
       
         
      •  
        Option 1: Set environment variables manually in your shell:
         
        export TF_VAR_project_id=\"your-google-cloud-project-id\"\nexport TF_VAR_region=\"your-preferred-region\"\nexport TF_VAR_github_owner=\"your-github-repo-owner\"\nexport TF_VAR_github_repo=\"your-github-repo-name\"\n
         
        •  
      •  
        Option 2: Create a terraform.tfvars file in the same directory as   your Terraform configuration and populate it with the following:
         
        project_id  = \"your-google-cloud-project-id\"\nregion      = \"your-preferred-region\"\ngithub_owner = \"your-github-repo-owner\"\ngithub_repo = \"your-github-repo-name\"\n
         
        •  
       
      3.  
    3.  
      Initialize and Apply Terraform: With the environment variables set,     initialize and apply the Terraform configuration:
       
      terraform init\nterraform apply\n
       
      Note: Applying Terraform may take a few minutes as it creates the necessary resources.
       
      4.  
    4.  
      Connect GitHub Repository to Cloud Build: Due to occasional issues with     automatic connections, you may need to manually attach your GitHub     repository to Cloud Build in the Google Cloud Console.
       
      If you get the following error you will need to manually connect your repository to the project:
       
      Error: Error creating Trigger: googleapi: Error 400: Repository mapping does\nnot exist.\n
       
      Re-run step 3 to ensure all resources are deployed
       
      5.  
    5.  
      Navigate to the Demo site: Once the Terraform setup is complete, switch     to the Demo site directory:
       
      cd platform-engineering/reference-architectures/cloud-deploy-flow/WebsiteDemo\n
       
      6.  
    6.  
      Authenticate and Run the Demo site:
       
         
      •  
        Ensure you are running these commands on a local machine or a machine   with GUI/web browser access, as Cloud Shell may not fully support   running the demo site.
         
        •  
      •  
        Set your Google Cloud project by running:
         
        gcloud config set project <your_project_id>\n
         
        •  
      •  
        Authenticate your Google Cloud CLI session:
         
        gcloud auth application-default login\n
         
        •  
      •  
        Install required npm packages and start the demo site:
         
        npm install\nnode index.js\n
         
        •  
      •  
        Open http://localhost:8080 in your browser to observe the demo site in   action.
         
        •  
       
      7.  
    7.  
      Trigger a Build in Cloud Build:
       
         
      • Initiate a build in Cloud Build. As the build progresses, messages will   display on the demo site, allowing you to follow each step in the   deployment process.
        •  
      • You can also monitor the deployment rollout on   Google Cloud Console.
        •  
       
      8.  
    8.  
      Approve the Rollout: When an approval message is received, you\u2019ll need     to send a response to complete the deployment. Use the message data provided     and add a ManualApproval field:
       
      {\n    \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n        \"Action\": \"Required\",\n        \"Rollout\": \"rollout-123\",\n        \"ReleaseId\": \"release-456\",\n        \"ManualApproval\": \"true\"\n    }\n    }\n}\n
       
      9.  
    9.  
      Verify the Deployment: Once the approval is processed, the deployment     should finish rolling out. Check the Cloud Deploy dashboard in the Google     Cloud Console to confirm the deployment status.
       
      10.  
    "},{"location":"reference-architectures/cloud_deploy_flow/#conclusion","title":"Conclusion","text":"
    This demo encapsulates the essential components and workflow for deploying applications using platform engineering practices. It illustrates how various services interact to ensure a smooth deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/","title":"Cloud Deployment Approvals with Pub/Sub","text":"
    This project provides a Google Cloud Run Function to automate deployment approvals based on messages received via Google Cloud Pub/Sub. The function processes deployment requests, checks conditions for rollout approval, and publishes an approval command if the requirements are met.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#features","title":"Features","text":"
       
    • Listens to Pub/Sub messages for deployment approvals
      •  
    • Validates deployment conditions (manual approval, rollout ID, etc.)
      •  
    • Publishes approval commands to another Pub/Sub topic if conditions are met
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#requirements","title":"Requirements","text":"
       
    • POSIX compliant Bash Shell
      •  
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Pub/Sub
      •  
    • Environment variables to configure project details
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Enable APIs: Enable the Google Cloud Pub/Sub and Deploy APIs for your     project:
       
      gcloud services enable pubsub.googleapis.com deploy.googleapis.com\n
       
      3.  
    3.  
      Deploy the Function: Use Google Cloud SDK to deploy the function:
       
      gcloud functions deploy cloudDeployApprovals --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_SUBSCRIBE_TOPIC\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#code-structure","title":"Code Structure","text":"
       
    •  
      config struct: Holds configuration for the environment variables.
       
      •  
    •  
      PubsubMessage and ApprovalsData structs: Define the structure of messages   received from Pub/Sub and attributes within them.
       
      •  
    •  
      cloudDeployApprovals function: Entry point for handling messages.   Validates the conditions and, if met, triggers the sendCommandPubSub   function to send an approval command.
       
      •  
    •  
      sendCommandPubSub function: Publishes a command message to the Pub/Sub   topic to approve a deployment rollout.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#usage","title":"Usage","text":"
    The function cloudDeployApprovals is invoked whenever a message is published to the configured Pub/Sub topic. Upon receiving a message, the function will:
     
       
    1. Parse and validate the message.
      2.  
    2. Check if the action is Required, if a rollout ID is provided, and if     manual approval is marked as \"true.\"
      3.  
    3. If conditions are met, it will publish an approval command to the     SENDTOPICID topic.
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    A message sent to the function should resemble this JSON structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded data>\",\n    \"attributes\": {\n      \"Action\": \"Required\",\n      \"Rollout\": \"rollout-123\",\n      \"ReleaseId\": \"release-456\",\n      \"ManualApproval\": \"true\"\n    }\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#custom-manual-approval-field","title":"Custom Manual Approval Field","text":"
    In the ApprovalsData struct, there is a ManualApproval field. This field is a custom addition, not provided by Google Cloud Deploy, and serves as a placeholder for an external approval system.
     
    To integrate the approval system, you can replace or adapt this field to suit your existing change process workflow. For instance, you could link this field to an external ticketing or project management system to track and verify approvals. Implementing an approval system allows greater control over deployment rollouts, ensuring they align with your organization\u2019s policies.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/#logging","title":"Logging","text":"
    The function logs each major step, from invocation to message processing and condition checking, to facilitate debugging and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/","title":"Cloud Deploy Interactions with Pub/Sub","text":"
    This project demonstrates a Google Cloud Run Function to manage deployments by creating releases, rollouts, or approving rollouts based on incoming Pub/Sub messages. The function leverages Google Cloud Deploy and listens for deployment-related commands sent via Pub/Sub, executing appropriate actions based on the command type.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#features","title":"Features","text":"
       
    •  
      Listens for Pub/Sub messages with deployment commands (CreateRelease,   CreateRollout, ApproveRollout) Messages should include protobuf request.
       
      •  
    •  
      Initiates Google Cloud Deploy actions based on the received command.
       
      •  
    •  
      Logs each step of the deployment process for better traceability.
       
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#setup","title":"Setup","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#requirements","title":"Requirements","text":"
       
    • Go 1.16 or later
      •  
    • Google Cloud SDK
      •  
    • Access to Google Cloud Deploy and Pub/Sub
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#installation","title":"Installation","text":"
       
    1.  
      Clone the repository:
       
      git clone <repository-url>\ncd <repository-folder>\n
       
      2.  
    2.  
      Set up Google Cloud: Ensure you have enabled the Google Cloud Deploy and     Pub/Sub APIs in your project.
       
      3.  
    3.  
      Deploy the Function: Deploy the function using Google Cloud SDK:
       
      gcloud functions deploy cloudDeployInteractions --runtime go116 \\\n--trigger-event-type google.cloud.pubsub.topic.v1.messagePublished \\\n--trigger-resource YOUR_TOPIC_NAME\n
       
      4.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#pubsub-message-format","title":"Pub/Sub Message Format","text":"
    The Pub/Sub message should include a JSON payload with a command field specifying the type of deployment action to execute. Examples of the command types include:
     
       
    • CreateRelease: Creates a new release for deployment.
      •  
    • CreateRollout: Initiates a rollout of the release.
      •  
    • ApproveRollout: Approves a pending rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#sample-pubsub-message","title":"Sample Pub/Sub Message","text":"
    The message should follow this structure:
     
    {\n  \"message\": {\n    \"data\": \"<base64-encoded JSON containing command data>\"\n  }\n}\n
     
    The JSON inside data should follow the format for DeployCommand:
     
    {\n  \"command\": \"CreateRelease\",\n  \"createReleaseRequest\": {\n    // Release creation parameters\n  },\n  \"createRolloutRequest\": {\n    // Rollout creation parameters\n  },\n  \"approveRolloutRequest\": {\n    // Rollout approval parameters\n  }\n}\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#code-structure","title":"Code Structure","text":"
       
    •  
      DeployCommand struct: Defines the command to be executed and the   parameters for each deploy action (create release, create rollout, or approve   rollout).
       
      •  
    •  
      cloudDeployInteractions function: Main function triggered by Pub/Sub   messages. It parses the message and calls the respective deployment function   based on the command.
       
      •  
    •  
      cdCreateRelease: Creates a release in Google Cloud Deploy.
       
      •  
    • cdCreateRollout: Initiates a rollout for a specified release.
      •  
    • cdApproveRollout: Approves an existing rollout.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/#logging","title":"Logging","text":"
    Each function logs key steps, from initialization to message handling and completion of deployments, helping in troubleshooting and monitoring.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/","title":"Cloud Deploy Operations Function","text":"
    This project contains a Google Cloud Run Function written in Go, designed to interact with Google Cloud Deploy. The function listens for deployment events on a Pub/Sub topic, processes those events, and triggers specific deployment operations based on the event details. For instance, when a deployment release succeeds, it triggers a rollout creation and sends the relevant command to another Pub/Sub topic.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#requirements","title":"Requirements","text":"
       
    • Go 1.20 or later
      •  
    • Google Cloud SDK
      •  
    • Google Cloud Pub/Sub
      •  
    • Google Cloud Deploy API
      •  
    • Set environment variables for Google Cloud project configuration
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#structure","title":"Structure","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#main-components","title":"Main Components","text":"
       
    • config: Stores the environment configuration necessary for the function.
      •  
    • PubsubMessage: Structure representing a message from Pub/Sub, with Data   payload and Attributes metadata.
      •  
    • OperationsData: Metadata that describes deployment action and resource   details.
      •  
    • CommandMessage: Structure for deployment commands, like CreateRollout.
      •  
    • cloudDeployOperations: Main Cloud Run Function triggered by a deployment   event, processes release successes to initiate rollouts.
      •  
    • sendCommandPubSub: Publishes a CommandMessage to a specified Pub/Sub   topic, which triggers deployment operations.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#function-workflow","title":"Function Workflow","text":"
       
    1. Trigger: The function cloudDeployOperations is triggered by a     deployment event, specifically a CloudEvent.
      2.  
    2. Event Parsing: The function parses the event data into a Message     struct, checking for deployment success events.
      3.  
    3. Rollout Creation: If a release success is detected, it creates a     CommandMessage for a rollout and calls sendCommandPubSub.
      4.  
    4. Command Publish: The sendCommandPubSub function publishes the     CommandMessage to a designated Pub/Sub topic to initiate the rollout.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#setup-and-deployment","title":"Setup and Deployment","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#local-development","title":"Local Development","text":"
       
    1. Clone the repository and set up your local environment with the necessary     environment variables.
      2.  
    2. Run the Cloud Run Functions framework locally to test the function:
      3.  
     
    functions-framework --target=cloudDeployOperations\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#deployment-to-google-cloud-run-functions","title":"Deployment to Google Cloud Run Functions","text":"
       
    1.  
      Set up your Google Cloud environment and enable the necessary APIs:
       
      gcloud services enable cloudfunctions.googleapis.com pubsub.googleapis.com\nclouddeploy.googleapis.com\n
       
      2.  
    2.  
      Deploy the function to Google Cloud:
       
      gcloud functions deploy cloudDeployOperations \\\n   --runtime go120 \\\n   --trigger-topic <YOUR_TRIGGER_TOPIC> \\\n   --set-env-vars PROJECTID=<YOUR_PROJECT_ID>,LOCATION=<YOUR_LOCATION>,SENDTOPICID=<YOUR_SEND_TOPIC_ID>\n
       
      3.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#error-handling","title":"Error Handling","text":"
       
    • If message parsing fails, the function logs an error but acknowledges the   message to prevent retries.
      •  
    • Command failures are logged, and the function acknowledges the message to   prevent reprocessing of erroneous commands.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#license","title":"License","text":"
    This project is licensed under the MIT License. See the LICENSE file for details.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/#notes","title":"Notes","text":"
       
    • For production environments, consider validating that the TargetId within   CommandMessage is dynamically populated based on actual Pub/Sub message   data.
      •  
    • The function relies on pubsub.NewClient which should be carefully monitored   in production for connection management.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/","title":"Example Cloud Run Function","text":"
    This project demonstrates a Google Cloud Run Function that triggers deployments based on Pub/Sub messages. The function listens for build notifications from Google Cloud Build and initiates a release in Google Cloud Deploy when a build succeeds.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#table-of-contents","title":"Table of Contents","text":"
       
    • Prerequisites
      •  
    • Env
      •  
    • Function Overview
      •  
    • Deploying the Function
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#prerequisites","title":"Prerequisites","text":"
       
    • Go version 1.15 or later
      •  
    • Google Cloud account
      •  
    • Google Cloud SDK installed and configured
      •  
    • Necessary permissions for Cloud Build and Cloud Deploy
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#environment-variables","title":"Environment Variables","text":"
    The function relies on environment variables to specify project configuration. Ensure these are set before deploying the function:
     Variable Name Description Required PROJECTID Google Cloud project ID Yes LOCATION The deployment location (region) Yes PIPELINE The name of the delivery pipeline in Cloud Deploy. Yes TRIGGER The ID of the build trigger in Cloud Build. Yes SENDTOPICID Pub/Sub topic ID for sending commands Yes"},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#function-overview","title":"Function Overview","text":"
    The deployTrigger function is invoked by Pub/Sub events. Here's a breakdown of its key components:
     
       
    1.  
      Initialization:
       
         
      • Loads environment variables into a configuration struct.
        •  
      • Registers the function to be triggered by CloudEvents.
        •  
       
      2.  
    2.  
      Message Handling:
       
         
      • Parses incoming Pub/Sub messages.
        •  
      • Validates build notifications based on specified criteria (trigger ID and   build status).
        •  
       
      3.  
    3.  
      Release Creation:
       
         
      • Extracts relevant image information from the build notification.
        •  
      • Constructs a CreateReleaseRequest for Cloud Deploy.
        •  
      • Sends the request to the specified Pub/Sub topic.
        •  
       
      4.  
    4.  
      Random ID Generation:
       
         
      • Generates a unique release ID to ensure each deployment is distinct.
        •  
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/#deploying-the-function","title":"Deploying the Function","text":"
    To deploy the function, follow these steps:
     
       
    1. Ensure that your Google Cloud SDK is authenticated and configured with the     correct project.
      2.  
    2. Use the following command to deploy the function:
      3.  
     
    gcloud functions deploy deployTrigger \\\n    --runtime go113 \\\n    --trigger-topic YOUR_TOPIC_NAME \\\n    --env-file .env\n
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/","title":"Random Date Service","text":"
    This repository contains a sample application designed to demonstrate how deployments can work through Google Cloud Deploy and Cloud Build. Instead of a traditional \"Hello World\" application, this project generates and serves a random date, showcasing how to set up a cloud-based service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage-note","title":"Usage Note","text":"
    This code is designed to integrate with the Terraform configuration for the cloud_deploy_flow demo. While you can deploy this component individually, it's primarily intended to be used as part of the full Terraform-managed workflow. Please note that this section of the readme may be less actively maintained, as the preferred deployment method relies on the Terraform setup.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#overview","title":"Overview","text":"
    The Random Date Service is built to illustrate the process of deploying an application using Cloud Run and Cloud Deploy. The application serves a random date formatted as a string. This simple service allows you to explore key concepts in cloud deployment without the complexity of a full-fledged application.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#components","title":"Components","text":""},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#1-maingo","title":"1. main.go","text":"
    This is the core of the application, where the HTTP server is defined. It handles requests and responds with a randomly generated date.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#2-dockerfile","title":"2. Dockerfile","text":"
    The Dockerfile specifies how to build a container image for the application. This image will be used in Cloud Run for deploying the service.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#3-skaffoldyaml","title":"3. skaffold.yaml","text":"
    This file is configured for Google Cloud Deploy, facilitating the deployment process by managing builds and configurations in a single file.
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#4-runyaml","title":"4. run.yaml","text":"
    The run.yaml file defines the configuration for Cloud Run and Cloud Deploy. Key aspects to note include:
     
       
    • Service Name: This defines the name of the service as   random-date-service.
      •  
    • Image Specification: The image field under spec is set to pizza.   This is crucial, as it indicates to Cloud Deploy where to substitute the   image. This substitution occurs based on the createRelease function in   main.go, specifically noted on line 122.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#usage","title":"Usage","text":"
    To deploy and test this application:
     
       
    1. Build the Docker Image: Use the provided Dockerfile to create a     container image.
      2.  
    2. Deploy to Cloud Run: Utilize the run.yaml configuration to deploy the     service.
      3.  
    3. Monitor Deployments: Use Cloud Deploy to observe the deployment pipeline     and ensure the service is running as expected.
      4.  
    4. Access the Service: After deployment, access the service through its     endpoint to receive a random date.
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/CloudRun/#conclusion","title":"Conclusion","text":"
    This sample application serves as a foundational example of how to leverage cloud services for deploying applications. By utilizing Google Cloud Deploy and Cloud Build, you can understand the deployment lifecycle and how cloud-native applications can be effectively managed and served.
     
    Feel free to explore the code and configurations provided in this repository to get a better grasp of the deployment process.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/","title":"Pub/Sub Local Demo","text":"
    This project is a simple demonstration of a Pub/Sub system using Google Cloud Pub/Sub and a basic Express.js server. It is designed to visually understand how messages are sent to and from Pub/Sub queues. The code provided is primarily for demonstration purposes and is not intended for production use.
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#features","title":"Features","text":"
       
    • Real-time Message Display: Messages from various Pub/Sub subscriptions are   fetched and displayed in a web interface.
      •  
    • Clear Messages: Users can clear all displayed messages from the UI.
      •  
    • Send Messages: Users can send messages to the Pub/Sub topic via the input   area.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#project-structure","title":"Project Structure","text":"
       
    • public/index.html: The main HTML file that provides the structure for the   web interface.
      •  
    • public/script.js: Contains JavaScript logic for fetching messages, sending   new messages, and clearing messages.
      •  
    • public/styles.css: Contains styling for the web interface to ensure a   clean and responsive layout.
      •  
    • index.js: The main server file that handles Pub/Sub operations and serves   static files.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#installation","title":"Installation","text":"
       
    1.  
      Install the required dependencies:
       
      npm install
       
      2.  
    2.  
      Set your Google Cloud project ID and subscription names in the index.js     file.
       
      3.  
    3.  
      Start the server:
       
      node index.js
       
      4.  
    4.  
      Open your web browser and go to http://localhost:8080 to access the demo.
       
      5.  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#usage","title":"Usage","text":"
       
    • Sending Messages: Enter a message in the textarea and click \"Submit\" to   send it to the Pub/Sub topic.
      •  
    • Viewing Messages: The messages received from the Pub/Sub subscriptions   will be displayed in their respective boxes.
      •  
    • Clearing Messages: Click the \"Clear\" button to remove all messages from   the display.
      •  
    "},{"location":"reference-architectures/cloud_deploy_flow/WebsiteDemo/#disclaimer","title":"Disclaimer","text":"
    This code is intended for educational and demonstration purposes only. It may not be suitable for production environments due to lack of error handling, security considerations, and scalability.
    "},{"location":"reference-architectures/github-runners-gke/","title":"Reference Guide: Deploy and use GitHub Actions Runners on GKE","text":""},{"location":"reference-architectures/github-runners-gke/#overview","title":"Overview","text":"
    This guide walks you through the process of setting up self-hosted GitHub Actions Runners on Google Kubernetes Engine (GKE) using the Terraform module terraform-google-github-actions-runners. It then provides instructions on how to create a basic GitHub Actions workflow to leverage these runners.
     
    "},{"location":"reference-architectures/github-runners-gke/#prerequisites","title":"Prerequisites","text":"
       
    • Terraform: Install Terraform on your local machine or use Cloud Shell
      •  
    • Google Cloud Project: Have a Google Cloud project with a Billing Account   linked and the following APIs enabled:
         
      • Cloud Resource Manager API cloudresourcemanager.googleapis.com
        •  
      • Identity and Access Management API iam.googleapis.com
        •  
      • Kubernetes Engine API container.googleapis.com
        •  
      • Service Usage API serviceusage.googleapis.com
        •  
       
      •  
    • GitHub Account: Have a GitHub organization, either personal or enterprise,   where you have administrator access.
      •  
     
    Run the following command to enable the prerequisite APIs:
     
    gcloud services enable \\\n  cloudresourcemanager.googleapis.com \\\n  iam.googleapis.com \\\n  container.googleapis.com \\\n  serviceusage.googleapis.com \\\n  --project <YOUR_PROJECT_ID>\n
    "},{"location":"reference-architectures/github-runners-gke/#register-a-github-app-for-authenticating-arc","title":"Register a GitHub App for Authenticating ARC","text":"
    Using a GitHub App for authentication allows you to make your self-hosted runners available to a GitHub organization that you own or have administrative access to. For more details on registering GitHub Apps, see GitHub\u2019s documentation.
     
    You will need 3 values from this section to use as inputs in the Terraform module:
     
       
    • GitHub App ID
      •  
    • GitHub App Private Key
      •  
    • GitHub App Installation ID
      •  
    "},{"location":"reference-architectures/github-runners-gke/#navigate-to-your-organization-github-app-settings","title":"Navigate to your Organization GitHub App settings","text":"
       
    1. Click your profile picture in the top-right
      2.  
    2. Click Your organizations
      3.  
    3. Select the organization you want to use for this walkthrough
      4.  
    4. Click Settings
      5.  
    5. Click \\<> Developer settings
      6.  
    6. Click GitHub Apps
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#create-a-new-github-app","title":"Create a new GitHub App","text":"
       
    1. Click New GitHub App
      2.  
    2. Under \u201cGitHub App name\u201d, choose a unique name such as \u201cmy-gke-arc-app\u201d
      3.  
    3. Under \u201cHomepage URL\u201d enter     https://github.com/actions/actions-runner-controller
      4.  
    4. Under \u201cWebhook,\u201d uncheck Active.
      5.  
    5. Under \u201cPermissions,\u201d click Repository permissions and use the dropdown     menu to select the following permissions:
         
      1. Metadata: Read-only
        2.  
       
      6.  
    6. Under \u201cPermissions,\u201d click Organization permissions and use the dropdown     menu to select the following permissions:
         
      1. Self-hosted runners: Read and write
        2.  
       
      7.  
    7. Click the Create GitHub App button
      8.  
    "},{"location":"reference-architectures/github-runners-gke/#gather-required-ids-and-keys","title":"Gather required IDs and keys","text":"
       
    1. On the GitHub App\u2019s page, save the value for \u201cApp ID\u201d
         
      1. You will use this as the value for gh_app_id in the Terraform module
        2.  
       
      2.  
    2. Under \u201cPrivate keys\u201d click Generate a private key. Save the .pem file     for later.
         
      1. You will use this as the value for gh_app_private_key in the Terraform     module
        2.  
       
      3.  
    3. In the menu at the top-left corner of the page, click Install App, and     next to your organization, click Install to install the app on your     organization.
         
      1. Choose All repositories to allow any repository in your org to have     access to your runners
        2.  
      2. Choose Only select repositories to allow specific repos to have     access to your runners
        3.  
       
      4.  
    4. Note the app installation ID, which you can find on the app installation     page, which has the following URL format:     https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID
         
      1. You will use this as the value for gh_app_installation_id in the     Terraform module.
        2.  
       
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#configure-terraform-example","title":"Configure Terraform example","text":""},{"location":"reference-architectures/github-runners-gke/#open-the-terraform-example","title":"Open the Terraform example","text":"
    Open the Terraform module repository in Cloud Shell automatically by clicking the button:
     
     
    Clicking this button will clone the repository into Cloud Shell, change into the example directory, and open the main.tf file in the Cloud Shell Editor.
    "},{"location":"reference-architectures/github-runners-gke/#modify-terraform-example-variables","title":"Modify Terraform example variables","text":"
       
    1. Insert your Google Cloud Project ID as the value of project_id
      2.  
    2. Modify the sample values of the following variables with the values you     saved from earlier.
         
      1. gh_app_id: insert the value of the App ID from the GitHub App page
        2.  
      2. gh_app_installation_id: insert the value from the URL of the app     installation page
        3.  
      3. gh_app_private_key:
           
        1. Copy the .pem file to example directory, alongside the main.tf     file
          2.  
        2. Insert the .pem filename you downloaded after generating the     private key for the app, like so:
             
          1. gh_app_private_key = file(\"example.private-key.pem\")
            2.  
           
          3.  
        3. Warning: Terraform will store the private key in state as plaintext.     It\u2019s recommended to secure your state file by using a backend such     as a GCS bucket with encryption. You can do so by following     these instructions.
          4.  
         
        4.  
       
      3.  
    3. Modify the value of gh_config_url with the URL of your GitHub     organization. It will be in the format of https://github.com/ORGANIZATION
      4.  
    4. (Optional) Specify any other parameters that you wish. For a full list of     variables you can modify, refer to the     module documentation.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#deploy-the-example","title":"Deploy the example","text":"
       
    1. Initialize Terraform: Run terraform init to download the required     providers.
      2.  
    2. Plan: Run terraform plan to preview the changes that will be made.
      3.  
    3. Apply: Run terraform apply and confirm to create the resources.
      4.  
     
    You will see the runners become available in your GitHub Organization:
     
       
    1. Go to your GitHub organization page
      2.  
    2. Click Settings
      3.  
    3. Open the \u201cActions\u201d drop-down in the left menu and choose Runners
      4.  
     
    You should see the runners appear as \u201carc-runners\u201d
    "},{"location":"reference-architectures/github-runners-gke/#creating-a-github-actions-workflow","title":"Creating a GitHub Actions Workflow","text":"
       
    1. Create a new GitHub repository within your organization.
      2.  
    2. In your GitHub repository, click the Actions tab.
      3.  
    3. Click New workflow
      4.  
    4. Under \u201cChoose workflow\u201d click set up a workflow yourself
      5.  
    5.  
      Paste the following configuration into the text editor:
       
      name: Actions Runner Controller Demo\non:\nworkflow_dispatch:\njobs:\nExplore-GitHub-Actions:\n   runs-on: arc-runners\n   steps:\n   - run: echo \"This job uses runner scale set runners!\"\n
       
      6.  
    6.  
      Click Commit changes to save the workflow to your repository.
       
      7.  
    "},{"location":"reference-architectures/github-runners-gke/#test-the-github-actions-workflow","title":"Test the GitHub Actions Workflow","text":"
       
    1. Go back to the Actions tab in your repository.
      2.  
    2. In the left menu, select the name of your workflow. This should be \u201cActions     Runner Controller Demo\u201d if you left the above configuration unchanged
      3.  
    3. Click Run workflow to open the drop-down menu, and click Run     workflow
      4.  
    4. The sample workflow executes on your GKE-hosted ARC runner set. You can view     the output within the GitHub Actions run history.
      5.  
    "},{"location":"reference-architectures/github-runners-gke/#cleanup","title":"Cleanup","text":""},{"location":"reference-architectures/github-runners-gke/#teardown-terraform-managed-infrastructure","title":"Teardown Terraform-managed infrastructure","text":"
       
    1.  
      Navigate back into the example directory you previously ran     terraform apply
       
      cd terraform-google-github-actions-runners/examples/gh-runner-gke-simple/\n
       
      2.  
    2.  
      Destroy Terraform-managed infrastructure
       
      terraform destroy\n
       
      3.  
     
    Warning: this will destroy the GKE cluster, example VPC, service accounts, and the Helm-managed workloads previously deployed by this example.
    "},{"location":"reference-architectures/github-runners-gke/#delete-github-resources","title":"Delete GitHub resources","text":"
    If you created a new GitHub App for testing purposes of this walkthrough, you can delete it via the following instructions. Note that any services authenticating via this GitHub App will lose access.
     
       
    1. Navigate to your Organization GitHub App settings
         
      1. Click your profile picture in the top-right
        2.  
      2. Click Your organizations
        3.  
      3. Select the organization you used for this walkthrough
        4.  
      4. Click Settings
        5.  
      5. Click the \\<> Developer settings drop-down
        6.  
      6. Click GitHub Apps
        7.  
       
      2.  
    2. In the row where your GitHub App is listed, click Edit
      3.  
    3. In the left-side menu, click Advanced
      4.  
    4. Click Delete GitHub App
      5.  
    5. Type the name of the GitHub App to confirm and delete.
      6.  
    "},{"location":"reference-architectures/sandboxes/","title":"Sandbox Projects Reference Architecure","text":"
    This architecture demonstrates how you can automate the provisioning of sandbox projects and automatically apply sensible guardrails and constraints. A sandbox project allows engineers to experiment with new technologies. Sandboxes are provisioned for a short period of time and with budget constraints.
    "},{"location":"reference-architectures/sandboxes/#architecture","title":"Architecture","text":"
    The following diagram is the high-level architecture for enabling self-service creation of sandbox projects.
     
     
       
    1. The system project contains the state database and infrastructure required     to create, delete and manage the lifecycle of the sandboxes.
      2.  
    2. User interface that engineers use to request and manage the sandboxes they     own.
      3.  
    3. Firestore stores the state of the overall environment. Documents in the     database represent all the active and inactive sandboxes. The document model     is detailed in the sandbox-modules readme.
      4.  
    4. Firestore triggers are Cloud Run functions whenever a document is created or     updated. Create and update events are handled by Cloud Run functions     onCreate and onModify. The functions contain the logic to decide if a     sandbox should be created or deleted.
      5.  
    5. infraManagerProcessor is a Cloud Run service that works with     Infrastructure Manager to kick off and monitor the infrastructure     management. This is handled in a Cloud Run service because the execution of     Terraform is a long running process.
      6.  
    6. Cloud Storage contains the Terraform templates and state used by     Infrastructure Manager.
      7.  
    7. Cloud Scheduler triggers the execution of sandbox lifecycle management     processes, for example a function that check for the expiration of sandboxes     and marking them for deletion.
      8.  
    "},{"location":"reference-architectures/sandboxes/#structure-of-the-repository","title":"Structure of the Repository","text":"
    This repository contains the code to stand up the reference architecture and also create difference sandbox templates in the catalog. This section describes the structure of the repository so you can better navigate the code.
    "},{"location":"reference-architectures/sandboxes/#examples","title":"Examples","text":"
    The /examples directory contains a sample Terraform deployment for deploying the reference architecture and command-line tool to exercise the automated creation of developer sandboxes. The examples are intended to provide you a starting point so you can incorporate the reference architecure into your infrastructure.
    "},{"location":"reference-architectures/sandboxes/#gcp-sandboxes","title":"GCP Sandboxes","text":"
    This example uses the Terraform modules from /sandbox-modules to deploy the reference architecture and includes instructions on how to get started.
    "},{"location":"reference-architectures/sandboxes/#command-line-interface-cli","title":"Command Line Interface (CLI)","text":"
    The workflows and lifecycle of the sandboxes deployed via the reference architecture are managed through the document model stored in Cloud Firestore. This abstraction has the benefit of separating the core logic included in the reference archiecture from the user experience (UX). As such the example command line interface lets you experiment with the reference architecture and learn about the object model.
    "},{"location":"reference-architectures/sandboxes/#catalog","title":"Catalog","text":"
    This directory contains a collection (catalog) of templates that you can use to deploy sandboxes. The reference architecture includes one for an empty project, but others could be added to support more specialized roles such as database admins, AI engineers, etc.
    "},{"location":"reference-architectures/sandboxes/#sandbox-modules","title":"Sandbox Modules","text":"
    These modules use the fabric modules to create the system project. Each module represents a large component of the overall reference architecture and each component can be combined into the one system project or spread across different projects to help with separation of duties.
    "},{"location":"reference-architectures/sandboxes/#fabric-modules","title":"Fabric Modules","text":"
    These are the base Terraform modules adopted from the Cloud Fabric Foundation. The fabric foundation is intended to be vendored, so we have copied them here for repeatbility of the overall deployment of the reference architecture.
     
    We recommend that as you need additional modules for templates in the catalog that you start with and vendor the modules from the Cloud Foundation Fabric into this directory.
    "},{"location":"reference-architectures/sandboxes/examples/cli/","title":"Example Command Line Interface","text":""},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/","title":"Overview","text":"
    This directory contains Terraform configuration files that let you deploy the system project. This example is a good entry point for testing the reference architecture and learning how it can be incorportated into your own infrastructure as code processes.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#architecture","title":"Architecture","text":"
    For an explanation of the components of the sandboxes reference architecture and the interaction flow, read the main Architecture section.
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#before-you-begin","title":"Before you begin","text":"
    In this section you prepare a folder for deployment.
     
       
    1. Open the Cloud Console
      2.  
    2.  
      Activate Cloud Shell \\     At the bottom of the Cloud Console, a Cloud Shell     session starts and displays a command-line prompt.
       
      3.  
    3.  
      In Cloud Shell, clone this repository
       
      git clone https://github.com/GoogleCloudPlatform/platform-engineering.git\n
       
      4.  
    4.  
      Export variables for the working directories
       
      export SANDBOXES_DIR=\"$(pwd)/reference-architectures/examples/gcp-sandboxes\"\nexport SANDBOXES_CLI=\"$(pwd)/reference-architectures/examples/cli\"\n
       
      5.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#preparing-the-sandboxes-folder","title":"Preparing the Sandboxes Folder","text":"
    In this section you prepare your environment for deploying the system project.
     
       
    1.  
      Go to the Manage Resources page in the Cloud Console in     the IAM & Admin menu.
       
      2.  
    2.  
      Click Create folder, then choose Folder.
       
      3.  
    3.  
      Enter a name for your folder. This folder will be used to contain the system     and sandbox projects.
       
      4.  
    4.  
      Click Create
       
      5.  
    5.  
      Copy the folder ID from the Manage resources page, you will need this value     later for use as Terraform variable.
       
      6.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#deploying-the-reference-architecture","title":"Deploying the reference architecture","text":"
       
    1.  
      Set the project ID and region in the corresponding Terraform environment     variables
       
      export TF_VAR_billing_account=\"<your billing account id>\"\nexport TF_VAR_sandboxes_folder=\"folders/<folder id from step 5>\"\nexport TF_VAR_system_project_name=\"<name for the system project>\"\n
       
      2.  
    2.  
      Change directory into the Terraform example directory and initialize     Terraform.
       
      cd \"${SANDBOXES_DIR}\"\nterraform init\n
       
      3.  
    3.  
      Apply the configuration. Answer yes when prompted, after reviewing the     resources that Terraform intends to create.
       
      terraform apply\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#creating-a-sandbox","title":"Creating a sandbox","text":"
    Now that the system project has been deployed, create a sandbox using the example cli.
     
       
    1.  
      Change directory into the example command-line tool directory
       
      cd \"${SANDBOXES_DIR}\"\n
       
      2.  
    2.  
      Install there required Python libraries
       
      pip install -r requirements.txt\n
       
      3.  
    3.  
      Create a Sandbox using the cli
       
      python ./sandbox.py create \\\n--system=\"<name of your system project>\" \\\n--project_id=\"<name of the sandbox to create>\"\n
       
      4.  
    "},{"location":"reference-architectures/sandboxes/examples/gcp-sandboxes/#next-steps","title":"Next steps","text":"
    Your sandboxes infrastructure is ready, you may continue to use the example cli to create and delete sandboxes. At this point it is recommended that you:
     
       
    • Review the detailed object and operating model
      •  
    • Adapt the CLI to meet your organization's requirements
      •  
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/","title":"Sandbox Projects","text":""},{"location":"reference-architectures/sandboxes/sandbox-modules/#data-model","title":"Data Model","text":"
    Each document stored in Cloud Firestore represents a sandbox. The following sections document the fields and structure of those documents.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#deployment","title":"Deployment","text":"Field Type Description _updateSource string This describes the last process or tool used to update or create the deployment document. For example, the example python cli _updateSource is set to python and when the firestore-processor Cloud Run updates the document it is set to cloudrun. status string Status of the sandbox, this changes create and delete operations progress. Refer to Key Statuses for detailed definitions of the values. projectId string The project ID of the sandbox. templateName string The name of the Terraform template from the catalog that the sandbox is based on. deploymentState object<DeploymentState> State object for the sandbox deployment. Contains data such as budget, current spend, expiration date, etc.The state object is updated by and used by the various lifecycle functions. infraManagerDeploymentId string ID returned by Infrastructure Manager for the deployment. infraManagerResult object<DeploymentResponse> This is the response object returned from Infrastructure Manager deployment operation. userId string Unique identifier for the user which owns the sandbox deployment. createdAt string Timestamp that the sandbox record was created at. updatedAt string Timestamp that the sandbox record was last updated. variables object<Variables> List of variable supplied by the user, which are in turned used by the template to create the sandbox. auditLog array[string] List of messages that the system can add as an audit log."},{"location":"reference-architectures/sandboxes/sandbox-modules/#deploymentstate","title":"DeploymentState","text":"Field Type Description budgetLimit number Spend limit for the sandbox. currentSpend number Current spend for the sandbox. expiresAt string Time base expiration for the sandbox."},{"location":"reference-architectures/sandboxes/sandbox-modules/#variables","title":"Variables","text":"
    Collection of key-value pairs that are used in the Infrastructure Manager request, for use as the Terraform variable values.
    "},{"location":"reference-architectures/sandboxes/sandbox-modules/#key-statuses","title":"Key Statuses","text":"
    The following table describes important statuses that are used during the lifecycle of a deployment.
     Status Set By Handled By Meaning provision_requested User Interface firestore-functions The user has requested that a sandbox be provisioned. provision_pending infra-manager-processor infra-manager-processor Indicates the request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. provision_inprogress infra-manager-processor infra-manager-processor Indicates that the request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. provision_error infra-manager-processor infra-manager-processor The deployment process has failed with an error. provision_successful infra-manager-processor infra-manager-processor The deployment process has succeeded and the sandbox is available and running. delete_requested User Interface firestore-functions The user or lifecycle process has requested that a sandbox be deleted. delete_pending infra-manager-processor infra-manager-processor Indicates the delete request was received by the infra-manager-processor but the request hasn\u2019t yet been made to Infrastructure Manager. delete_inprogress infra-manager-processor infra-manager-processor Indicates that the delete request has been submitted to Infrastructure Manager and it is in progress with Infrastructure Manager. delete_error infra-manager-processor infra-manager-processor The delete process has failed with an error. delete_successful infra-manager-processor infra-manager-processor The delete process has succeeded."}]}
\ No newline at end of file
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index 45195732..040a5732 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -2,78 +2,78 @@
 
     
          https://googlecloudplatform.github.io/platform-engineering/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/code-of-conduct/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/contributing/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/accelerating-migrations/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/automated-password-rotation/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/backstage/backstage-quickstart/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployApprovals/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployInteractions/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/cloudDeployOperations/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudFunctions/createRelease/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/CloudRun/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/cloud_deploy_flow/WebsiteDemo/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/github-runners-gke/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/cli/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/examples/gcp-sandboxes/
-         2025-07-11
+         2025-07-16
     
     
          https://googlecloudplatform.github.io/platform-engineering/reference-architectures/sandboxes/sandbox-modules/
-         2025-07-11
+         2025-07-16
     
 
\ No newline at end of file
diff --git a/docs/sitemap.xml.gz b/docs/sitemap.xml.gz
index 706fbcda1ef76c6dd834857e84ffe8337b8651db..b39a843b4103379e0cce8dced05e33b30283e477 100644
GIT binary patch
literal 438
zcmV;n0ZINJiwFpS>vm`Y|8r?{Wo=<_E_iKh0Ns~OZ`&{ohVT6qf$y@?0_#4IxENLx
z=xIQ)VaGz28AYfpsw7os|9-NQ6}uJ1cBrO{X__XUm!K(mb9ETQ>IsNF#%*)nuA3D^
zi-V7D+kF3aqdqiO-PtBnV31V_j=XL5BI%RX(=@g01xAy&MRJ<$zTB`rIrX~z*mP&B
zvgQOwYgwCswwtqXN#hV*02?q5DHz!yjji*t&wcAr|8^;e&PM<$WO}3L1b^l;EF%ux
z#roo%TEAE4mnG!nI|Aruf`80XiUkUH%3>U{l{dx~g?yjotrd-62UHpjlp(t>?P4Vp
zur}6005UVM8oiq>j9dqM795Shpps#pkOoC4i-0Qldt)D1jDyN(pDp#6{bG_-Az?{!
zKR^oj^L`g_s+9W6?77L&9>3Kk;CgxfIVHj;6ZpSVb{nOPDO=BUdX2cR32GX=FH0jR
zbNR09EQXuiv_fSXQu!%#D2jI$R;b6uTmd^3D
gqC1;Zt@S@ys=(y%H}uP;{3>?809@{9mRb$~0KIzDd;kCd

literal 437
zcmV;m0ZRTKiwFn+RB&hl|8r?{Wo=<_E_iKh0Nt0tZrd;nhVOlfz;{_mfuY-wxENLx
z=&3-lVaGz28AYfpsw7osZ$H_|irt1`J5t+Sf
z;^3p(HDABns`t%xcd^YB7-Us~Bk!7nNcyPtG)*mgfzc#xk(_3GC^xLnPQ7kFG~LCj
ztU1BSTGlq8?dB|8(l|sHzy{1i3P$!wW9z&ea^HH?KV1r<^AUgwncnI-!LRuY%ZNjF
zxxRd-)>mq?DIsUy2|&LR{9~3h4%1g?x0j%G|@=^S4x
fy0b~uTK}`93QP`vL%&?guVVKDlE~*LS`GjJlq}hX