update yml and pipeline script

Dockerfile

@@ -14,7 +14,7 @@ FROM public.ecr.aws/lambda/python:3.12
 
 # Install git using dnf (https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-base)
 # For python 3.12, dnf replaces yum for package management
-RUN dnf install -y git-2.40.1 && dnf clean all
+RUN dnf install -y git-all && dnf update && dnf clean all 
 
 # Copy the poetry.lock and pyproject.toml files
 COPY pyproject.toml poetry.lock ${LAMBDA_TASK_ROOT}/

concourse/ci.yml

@@ -1,6 +1,11 @@
 ---
 # Kics linting thinks that the "password" and "github_access_token" keys are secrets, but they are not.
 # These instances are ignored appropriately.
+resource_types:
+  - name: key-value
+    type: registry-image
+    source:
+      repository: gstack/keyval-resource
 
 resources:
   - name: resource-repo
@@ -12,14 +17,113 @@ resources:
       # kics-scan ignore-line
       password: x-oauth-basic # checkov:skip=CKV_SECRET_6:Checkov thinks this is a secret, but it is not
       branch: ((branch))
+  - name: github-release
+    type: github-release
+    source:
+      owner: ONS-Innovation
+      repository: github-repository-archive-script
+      access_token: ((github_access_token))
+  - name: github-release-tag
+    type: key-value
+    source:
+      file: tag-output/tag
+
+# Define the common terraform task as an anchor
+terraform-task: &terraform-task
+  task: terraform-deploy
+  privileged: true
+  config:
+    platform: linux
+    image_resource:
+      type: docker-image
+      source:
+        repository: hashicorp/terraform
+    inputs:
+      - name: resource-repo
+      - name: github-release-tag
+    params:
+      secrets: ((sdp_((env))_github_repo_archive_secrets))
+      github_access_token: ((github_access_token))
+      env: ((env))
+      branch: ((branch))
+    run:
+      path: sh
+      args:
+        - -cx
+        - |
+          echo "DEBUG: Environment is ${env}"
+          echo "DEBUG: Tag is ${tag}"
+          export tag=$(cat github-release-tag/tag)
+          if [[ "$env" == "prod" ]] && [[ "$branch" != "main" && "$branch" != "master" ]]; then
+            echo "Not on main branch, skipping terraform for prod."
+            exit 0
+          fi
+          apk add --no-cache jq curl
+          if [[ "$env" == "prod" && ! "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "ERROR: Tag '$tag' is not in semantic versioning format (vX.Y.Z)"
+            exit 1
+          fi
+          chmod u+x ./resource-repo/concourse/scripts/terraform_infra.sh
+          ./resource-repo/concourse/scripts/terraform_infra.sh
+    timeout: 30m
+
 
 jobs:
-  - name: build-and-push
+  - name: calculate-tag
     public: true
     plan:
+      - get: github-release
+        trigger: true
       - get: resource-repo
-        timeout: 5m
-      - task: build-image
+        trigger: false
+      - task: calculate-tag
+        config:
+          platform: linux
+          image_resource:
+            type: docker-image
+            source:
+              repository: alpine
+          inputs:
+            - name: resource-repo
+            - name: github-release
+          outputs:
+            - name: tag-output # Output directory for the tag
+          params:
+            branch: ((branch))
+          run:
+            path: sh
+            args:
+              - -cx
+              - |
+                apk add --no-cache git
+                echo "Calculating tag for branch: ${branch}"
+                if [[ "$branch" == "main" || "$branch" == "master" ]]; then
+                  # Get the latest tag that matches the format vX.Y.Z
+                  tag=$(git -C resource-repo tag --list 'v[0-9]*.[0-9]*.[0-9]*' | sort -V | tail -n 1)
+                  if [[ -z "$tag" ]]; then
+                    echo "No valid semantic versioning tags (vX.Y.Z) found. Cannot set pipeline."
+                    exit 1
+                  fi
+                else
+                  # Remove non-alphanumeric characters and take the first 7 characters
+                  tag=$(echo "${branch}" | tr -cd '[:alnum:]' | cut -c1-7)
+                fi
+                echo "Calculated tag: ${tag}"
+                # Write the tag to a file for output
+                echo "${tag}" > tag-output/tag
+      - put: github-release-tag
+        params:
+          directory: tag-output
+
+  - name: deploy-after-github-release-dev
+    public: true
+    plan:
+      - get: github-release-tag
+        passed: [calculate-tag]
+        trigger: true
+      - get: resource-repo
+        trigger: false
+      - task: deploy-release
         privileged: true
         config:
           platform: linux
@@ -29,56 +133,153 @@ jobs:
               repository: hashicorp/terraform
           inputs:
             - name: resource-repo
+            - name: github-release-tag
           params:
-            aws_account_id: ((aws_account_sdp_((env))))
-            aws_role_arn: arn:aws:iam::((aws_account_sdp_((env)))):role/sdp-concourse-((env))
+            aws_account_id: ((aws_account_sdp_dev))
+            aws_role_arn: arn:aws:iam::((aws_account_sdp_dev)):role/sdp-concourse-dev
             secrets: ((sdp_((env))_github_repo_archive_secrets))
-          run: # binary used to build the image
+            env: dev
+            tag: github-release-tag/tag # Use the release tag from the resource
+            repo_name: ((repo_name))
+          run:
             path: sh
             args:
               - -cx
               - |
                 apk add --no-cache aws-cli podman jq iptables curl
-
-                if [[ "((env))" == "prod" ]]; then
-                  tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-repository-archive-script/releases" | jq -r '.[0].tag_name')
-                  export tag
-                else
-                  export tag=((tag))
-                fi
-                git rev-parse --abbrev-ref HEAD
+                export repo_name=((repo_name))
+                export tag=$(cat github-release-tag/tag)
+                # Write the tag to a file for output
+                echo "${tag}" > release-tag-output/tag
+                echo "Using tag: ${tag}"
                 chmod u+x ./resource-repo/concourse/scripts/assume_role.sh
                 chmod u+x ./resource-repo/concourse/scripts/build_image.sh
                 source ./resource-repo/concourse/scripts/assume_role.sh
                 ./resource-repo/concourse/scripts/build_image.sh
-        timeout: 10m
-      - task: terraform
+        timeout: 15m
+      - <<: *terraform-task
+        params:
+          github_access_token: ((github_access_token))
+          env: dev
+          secrets: ((sdp_dev_github_repo_archive_secrets))
+
+  - name: release-build-and-push-prod
+    public: true
+    plan:
+      - get: resource-repo
+        passed: [deploy-after-github-release-dev]
+        trigger: false # Manual trigger only
+        timeout: 5m
+      - get: github-release-tag
+        passed: [deploy-after-github-release-dev]
+      - task: build-image
         privileged: true
         config:
           platform: linux
           image_resource:
             type: docker-image
-            source: { repository: hashicorp/terraform }
+            source:
+              repository: hashicorp/terraform
           inputs:
             - name: resource-repo
+            - name: github-release-tag
           params:
-            secrets: ((sdp_((env))_github_repo_archive_secrets))
-            # kics-scan ignore-line
-            github_access_token: ((github_access_token))
+            aws_account_id: ((aws_account_sdp_prod))
+            aws_role_arn: arn:aws:iam::((aws_account_sdp_prod)):role/sdp-concourse-prod
+            secrets: ((sdp_prod_github_repo_archive_secrets))
+            env: prod
+            tag: github-release-tag/tag
+            repo_name: ((repo_name))
+            branch: ((branch))
           run:
             path: sh
             args:
               - -cx
               - |
-                apk add --no-cache jq curl
-
-                if [[ "((env))" == "prod" ]]; then
-                  tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-repository-archive-script/releases" | jq -r '.[0].tag_name')
-                  export tag
-                else
-                  export tag=((tag))
+                if [[ "$branch" != "main" ]]; then
+                  echo "Not on main branch, skipping build."
+                  exit 0
                 fi
-                chmod u+x ./resource-repo/concourse/scripts/terraform_infra.sh
-                export env=((env))
-                ./resource-repo/concourse/scripts/terraform_infra.sh
-        timeout: 30m
+                apk add --no-cache aws-cli podman jq iptables curl
+                export repo_name=((repo_name))
+                export tag=$(cat github-release-tag/tag)
+                echo "Using release tag: ${tag}"
+                chmod u+x ./resource-repo/concourse/scripts/assume_role.sh
+                chmod u+x ./resource-repo/concourse/scripts/build_image.sh
+                source ./resource-repo/concourse/scripts/assume_role.sh
+                ./resource-repo/concourse/scripts/build_image.sh
+        timeout: 10m
+      - <<: *terraform-task
+        params:
+          github_access_token: ((github_access_token))
+          env: prod
+          secrets: ((sdp_prod_github_repo_archive_secrets))
+
+  # - name: build-and-push
+  #   public: true
+  #   plan:
+  #     - get: resource-repo
+  #       timeout: 5m
+  #     - task: build-image
+  #       privileged: true
+  #       config:
+  #         platform: linux
+  #         image_resource:
+  #           type: docker-image
+  #           source:
+  #             repository: hashicorp/terraform
+  #         inputs:
+  #           - name: resource-repo
+  #         params:
+  #           aws_account_id: ((aws_account_sdp_((env))))
+  #           aws_role_arn: arn:aws:iam::((aws_account_sdp_((env)))):role/sdp-concourse-((env))
+  #           secrets: ((sdp_((env))_github_repo_archive_secrets))
+  #         run: # binary used to build the image
+  #           path: sh
+  #           args:
+  #             - -cx
+  #             - |
+  #               apk add --no-cache aws-cli podman jq iptables curl
+
+  #               if [[ "((env))" == "prod" ]]; then
+  #                 tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-repository-archive-script/releases" | jq -r '.[0].tag_name')
+  #                 export tag
+  #               else
+  #                 export tag=((tag))
+  #               fi
+  #               git rev-parse --abbrev-ref HEAD
+  #               chmod u+x ./resource-repo/concourse/scripts/assume_role.sh
+  #               chmod u+x ./resource-repo/concourse/scripts/build_image.sh
+  #               source ./resource-repo/concourse/scripts/assume_role.sh
+  #               ./resource-repo/concourse/scripts/build_image.sh
+  #       timeout: 10m
+      # - task: terraform
+      #   privileged: true
+      #   config:
+      #     platform: linux
+      #     image_resource:
+      #       type: docker-image
+      #       source: { repository: hashicorp/terraform }
+      #     inputs:
+      #       - name: resource-repo
+      #     params:
+      #       secrets: ((sdp_((env))_github_repo_archive_secrets))
+      #       # kics-scan ignore-line
+      #       github_access_token: ((github_access_token))
+      #     run:
+      #       path: sh
+      #       args:
+      #         - -cx
+      #         - |
+      #           apk add --no-cache jq curl
+
+      #           if [[ "((env))" == "prod" ]]; then
+      #             tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-repository-archive-script/releases" | jq -r '.[0].tag_name')
+      #             export tag
+      #           else
+      #             export tag=((tag))
+      #           fi
+      #           chmod u+x ./resource-repo/concourse/scripts/terraform_infra.sh
+      #           export env=((env))
+      #           ./resource-repo/concourse/scripts/terraform_infra.sh
+      #   timeout: 30m

concourse/scripts/set_pipeline.sh

@@ -1,29 +1,27 @@
 # shellcheck disable=SC3040,SC2154,SC2148
+#!/bin/bash
+set -eo pipefail
+# Usage: ./set_pipeline.sh
 
-repo_name=${1}
+# Define repository name
+repo_name="github-repository-archive-script"
 
-if [[ $# -gt 1 ]]; then
-    branch=${2}
-    git rev-parse --verify "${branch}"
-    # shellcheck disable=SC2181
-    if [[ $? -ne 0 ]]; then
-        echo "Branch \"${branch}\" does not exist"
-        exit 1
-    fi
-else
-    branch=$(git rev-parse --abbrev-ref HEAD)
-fi
+# Always use the current branch
+branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || { echo "Failed to get branch name"; exit 1; })
 
-if [[ ${branch} == "main" || ${branch} == "master" ]]; then
-    env="prod"
-else
-    env="dev"
+if ! git rev-parse --verify "${branch}" >/dev/null 2>&1; then
+    echo "Branch \"${branch}\" does not exist. Cannot set a pipeline without a valid branch."
+    exit 1
 fi
 
-if [[ ${env} == "dev" ]]; then
-    tag=$(git rev-parse HEAD)
+if [[ ${branch} == "main" || ${branch} == "master" ]]; then
+    pipeline_name=${repo_name}
 else
-    tag=$(git tag | tail -n 1)
+    # Remove non-alphanumeric characters and take the first 7 characters
+    sanitized_branch=$(echo "${branch}" | tr -cd '[:alnum:]' | cut -c1-7)
+    pipeline_name=${repo_name}-${sanitized_branch}
 fi
 
-fly -t aws-sdp set-pipeline -c concourse/ci.yml -p "${repo_name}"-"${branch}" -v branch="${branch}" -v tag="${tag}" -v env="${env}"
+# Set the Concourse pipeline
+fly -t aws-sdp set-pipeline -c concourse/ci.yml -p ${pipeline_name}  -v branch=${branch} -v repo_name=${repo_name} -v env=dev
+echo "Pipeline \"${pipeline_name}\" has been set successfully."

terraform/service/data.tf

@@ -87,3 +87,9 @@ data "aws_iam_policy_document" "lambda_eventbridge_policy" {
     ]
   }
 }
+
+# Resolve the pushed image (must exist before terraform apply)
+data "aws_ecr_image" "lambda_image" {
+  repository_name = data.aws_ecr_repository.profile_lambda_ecr_repo.name
+  image_tag       = var.lambda_version
+}

terraform/service/main.tf

@@ -27,7 +27,7 @@ resource "aws_security_group" "lambda_sg" {
 resource "aws_lambda_function" "lambda_function" {
   function_name = var.lambda_name
   timeout       = var.lambda_timeout
-  image_uri     = "${data.aws_ecr_repository.profile_lambda_ecr_repo.repository_url}:${var.lambda_version}"
+  image_uri     = "${data.aws_ecr_repository.profile_lambda_ecr_repo.repository_url}@${data.aws_ecr_image.lambda_image.image_digest}"
   package_type  = "Image"
   architectures = [var.lambda_arch]
   logging_config {

terraform/service/outputs.tf

@@ -21,4 +21,9 @@ output "repo_name" {
 output "rule_arn" {
   description = "ARN of the EventBridge rule"
   value       = module.eventbridge.eventbridge_rules["${var.lambda_name}-crons"]["arn"]
+}
+
+output "image_digest" {
+  description = "Digest of the Lambda function container image"
+  value       = data.aws_ecr_image.lambda_image.image_digest
 }