From e034c1467dd659709cceee33e3aae67895212f98 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 1 Dec 2024 23:10:17 +0100 Subject: [PATCH 001/148] iam --- modules/runner/iam.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/runner/iam.tf b/modules/runner/iam.tf index 2834513..ececd57 100644 --- a/modules/runner/iam.tf +++ b/modules/runner/iam.tf @@ -1,7 +1,7 @@ -# resource "aws_iam_instance_profile" "runner_instance_profile" { -# name = "runner_instance_profile" -# role = aws_iam_role.github_actions_assume_role.name -# } +resource "aws_iam_instance_profile" "runner_instance_profile" { + name = "runner_instance_profile" + role = aws_iam_role.github_actions_assume_role.name +} data "aws_iam_policy_document" "assume_role" { statement { From 4a24f5fe6fddc9b1163f7311c6a43e406d1bb4cb Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 1 Dec 2024 23:10:45 +0100 Subject: [PATCH 002/148] changing workflow trigger --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5b58a50..72f51d6 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,5 +1,5 @@ name: do-the-job -on: pull_request +on: push jobs: start-runner: name: Start self-hosted EC2 runner From cf54bd9e7d35a584f6ed5587ae4ac29d043bea36 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 1 Dec 2024 23:33:24 +0100 Subject: [PATCH 003/148] fix vpc id --- .github/workflows/runner.yaml | 2 +- main.tf | 2 +- modules/runner/main.tf | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 72f51d6..2504fb5 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -24,7 +24,7 @@ jobs: ec2-instance-type: t3.nano subnet-id: subnet-0458f5906f3685c08 security-group-id: sg-035e60401f1ac532a - iam-role-name: runner_policy # optional, requires additional permissions + iam-role-name: runner_instance_profile # optional, requires additional permissions aws-resource-tags: > # optional, requires additional permissions [ {"Key": "Name", "Value": "ec2-github-runner"}, diff --git a/main.tf b/main.tf index 7b434e1..fea9ae1 100644 --- a/main.tf +++ b/main.tf @@ -21,7 +21,7 @@ module "runner" { source = "./modules/runner" ami_id = var.ami_id - vpc_id = module.vpc.vpc_id + vpc_id = module.vpc.default_vpc_id runner_instance_type = "t2.micro" subnet_id = tostring(module.vpc.public_subnets[0]) default_tags = merge( diff --git a/modules/runner/main.tf b/modules/runner/main.tf index 803fe6c..13a42a4 100644 --- a/modules/runner/main.tf +++ b/modules/runner/main.tf @@ -13,6 +13,8 @@ resource "aws_instance" "gh-runner" { instance_type = var.runner_instance_type iam_instance_profile = aws_iam_instance_profile.runner_instance_profile.name user_data = file("${path.module}/userdata.sh") + + subnet_id = var.subnet_id vpc_security_group_ids = [aws_security_group.gh-runner.id] # network_interface { From a6ae584953fe155a172d7ee8c66d54d480aea617 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 1 Dec 2024 23:41:41 +0100 Subject: [PATCH 004/148] Changing SG --- .github/workflows/runner.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 2504fb5..505f163 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -23,7 +23,8 @@ jobs: ec2-image-id: ami-02141377eee7defb9 ec2-instance-type: t3.nano subnet-id: subnet-0458f5906f3685c08 - security-group-id: sg-035e60401f1ac532a + security-group-id: +sg-00ed4a21eea0e2bba iam-role-name: runner_instance_profile # optional, requires additional permissions aws-resource-tags: > # optional, requires additional permissions [ From 4149ecd41ce85ff8caa34cbf8aba7ecd49fe2d74 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 1 Dec 2024 23:43:07 +0100 Subject: [PATCH 005/148] Changing SG --- .github/workflows/runner.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 505f163..302fb6a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -23,8 +23,7 @@ jobs: ec2-image-id: ami-02141377eee7defb9 ec2-instance-type: t3.nano subnet-id: subnet-0458f5906f3685c08 - security-group-id: -sg-00ed4a21eea0e2bba + security-group-id: sg-00ed4a21eea0e2bba iam-role-name: runner_instance_profile # optional, requires additional permissions aws-resource-tags: > # optional, requires additional permissions [ From 12ccf061985d93e7bc817355df507b595c13a8f9 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Mon, 2 Dec 2024 01:20:05 +0100 Subject: [PATCH 006/148] test --- .github/workflows/runner.yaml | 93 ++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 302fb6a..4e88a8d 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -3,58 +3,59 @@ on: push jobs: start-runner: name: Start self-hosted EC2 runner - runs-on: ubuntu-latest + runs-on: aws-runner outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-02141377eee7defb9 - ec2-instance-type: t3.nano - subnet-id: subnet-0458f5906f3685c08 - security-group-id: sg-00ed4a21eea0e2bba - iam-role-name: runner_instance_profile # optional, requires additional permissions - aws-resource-tags: > # optional, requires additional permissions - [ - {"Key": "Name", "Value": "ec2-github-runner"}, - {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} - ] + # steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} + # - name: Start EC2 runner + # id: start-ec2-runner + # uses: machulav/ec2-github-runner@v2 + # with: + # mode: start + # github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + # ec2-image-id: ami-02141377eee7defb9 + # ec2-instance-type: t3.nano + # subnet-id: subnet-0458f5906f3685c08 + # security-group-id: sg-00ed4a21eea0e2bba + # iam-role-name: runner_instance_profile # optional, requires additional permissions + # aws-resource-tags: > # optional, requires additional permissions + # [ + # {"Key": "Name", "Value": "ec2-github-runner"}, + # {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} + # ] do-the-job: name: Do the job on the runner needs: start-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + # runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + runs-on: aws-runner steps: - name: Hello World run: echo 'Hello World!' - stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - do-the-job # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file + # stop-runner: + # name: Stop self-hosted EC2 runner + # needs: + # - start-runner # required to get output from the start-runner job + # - do-the-job # required to wait when the main job is done + # runs-on: ubuntu-latest + # if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + # steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} + # - name: Stop EC2 runner + # uses: machulav/ec2-github-runner@v2 + # with: + # mode: stop + # github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + # label: ${{ needs.start-runner.outputs.label }} + # ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file From cdb7d96403817fd2dbc91961f8f650b8fb65d3b8 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Mon, 2 Dec 2024 01:21:11 +0100 Subject: [PATCH 007/148] test --- .github/workflows/runner.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 4e88a8d..56e4366 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,12 +1,12 @@ name: do-the-job on: push jobs: - start-runner: - name: Start self-hosted EC2 runner - runs-on: aws-runner - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + # start-runner: + # name: Start self-hosted EC2 runner + # runs-on: aws-runner + # outputs: + # label: ${{ steps.start-ec2-runner.outputs.label }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} # steps: # - name: Configure AWS credentials # uses: aws-actions/configure-aws-credentials@v4 From 9ada3fde8f101ac0cc9851345e892cfcee7fe322 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Mon, 2 Dec 2024 01:21:52 +0100 Subject: [PATCH 008/148] test --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 56e4366..2ec175c 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -32,7 +32,7 @@ jobs: # ] do-the-job: name: Do the job on the runner - needs: start-runner # required to start the main job when the runner is ready + # needs: start-runner # required to start the main job when the runner is ready # runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner runs-on: aws-runner steps: From eb765b788b58f0fd97e2938f2e69bead4032bfd1 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Mon, 2 Dec 2024 01:34:01 +0100 Subject: [PATCH 009/148] test --- .github/workflows/runner.yaml | 119 +++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 2ec175c..05cdb37 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,61 +1,74 @@ name: do-the-job on: push jobs: - # start-runner: - # name: Start self-hosted EC2 runner - # runs-on: aws-runner - # outputs: - # label: ${{ steps.start-ec2-runner.outputs.label }} - # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - # steps: - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.AWS_REGION }} - # - name: Start EC2 runner - # id: start-ec2-runner - # uses: machulav/ec2-github-runner@v2 - # with: - # mode: start - # github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - # ec2-image-id: ami-02141377eee7defb9 - # ec2-instance-type: t3.nano - # subnet-id: subnet-0458f5906f3685c08 - # security-group-id: sg-00ed4a21eea0e2bba - # iam-role-name: runner_instance_profile # optional, requires additional permissions - # aws-resource-tags: > # optional, requires additional permissions - # [ - # {"Key": "Name", "Value": "ec2-github-runner"}, - # {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} - # ] do-the-job: name: Do the job on the runner - # needs: start-runner # required to start the main job when the runner is ready - # runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner runs-on: aws-runner steps: + - uses: actions/checkout@v2 - name: Hello World - run: echo 'Hello World!' - # stop-runner: - # name: Stop self-hosted EC2 runner - # needs: - # - start-runner # required to get output from the start-runner job - # - do-the-job # required to wait when the main job is done - # runs-on: ubuntu-latest - # if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - # steps: - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.AWS_REGION }} - # - name: Stop EC2 runner - # uses: machulav/ec2-github-runner@v2 - # with: - # mode: stop - # github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - # label: ${{ needs.start-runner.outputs.label }} - # ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file + run: | + echo 'Hello World!' + sleep 15 + echo 'Goodbye!' +#### +# name: do-the-job +# on: push +# jobs: +# start-runner: +# name: Start self-hosted EC2 runner +# runs-on: ubuntu-latest +# outputs: +# label: ${{ steps.start-ec2-runner.outputs.label }} +# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} +# steps: +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: ${{ secrets.AWS_REGION }} +# - name: Start EC2 runner +# id: start-ec2-runner +# uses: machulav/ec2-github-runner@v2 +# with: +# mode: start +# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} +# ec2-image-id: ami-0e9085e60087ce171 +# ec2-instance-type: t3.nano +# subnet-id: subnet-0a63815aa47fb2deb +# security-group-id: sg-0633be23a5467c59c +# iam-role-name: runner_instance_profile # optional, requires additional permissions +# aws-resource-tags: > # optional, requires additional permissions +# [ +# {"Key": "Name", "Value": "ec2-github-runner"}, +# {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} +# ] +# do-the-job: +# name: Do the job on the runner +# needs: start-runner # required to start the main job when the runner is ready +# runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner +# steps: +# - name: Hello World +# run: echo 'Hello World!' +# stop-runner: +# name: Stop self-hosted EC2 runner +# needs: +# - start-runner # required to get output from the start-runner job +# - do-the-job # required to wait when the main job is done +# runs-on: ubuntu-latest +# if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs +# steps: +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: ${{ secrets.AWS_REGION }} +# - name: Stop EC2 runner +# uses: machulav/ec2-github-runner@v2 +# with: +# mode: stop +# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} +# label: ${{ needs.start-runner.outputs.label }} +# ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file From 2bfc419391e04b07c5337a47244359af1c92af05 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Mon, 2 Dec 2024 01:39:15 +0100 Subject: [PATCH 010/148] test --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 05cdb37..0a82098 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -3,7 +3,7 @@ on: push jobs: do-the-job: name: Do the job on the runner - runs-on: aws-runner + runs-on: self-hosted steps: - uses: actions/checkout@v2 - name: Hello World From f857de47df41ae371b1a6ac35060a40a00e09ccf Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Mon, 2 Dec 2024 01:40:01 +0100 Subject: [PATCH 011/148] test --- .github/workflows/runner.yaml | 134 +++++++++++++++++----------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 0a82098..6268233 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,74 +1,74 @@ -name: do-the-job -on: push -jobs: - do-the-job: - name: Do the job on the runner - runs-on: self-hosted - steps: - - uses: actions/checkout@v2 - - name: Hello World - run: | - echo 'Hello World!' - sleep 15 - echo 'Goodbye!' -#### # name: do-the-job # on: push # jobs: -# start-runner: -# name: Start self-hosted EC2 runner -# runs-on: ubuntu-latest -# outputs: -# label: ${{ steps.start-ec2-runner.outputs.label }} -# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} -# steps: -# - name: Configure AWS credentials -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: ${{ secrets.AWS_REGION }} -# - name: Start EC2 runner -# id: start-ec2-runner -# uses: machulav/ec2-github-runner@v2 -# with: -# mode: start -# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} -# ec2-image-id: ami-0e9085e60087ce171 -# ec2-instance-type: t3.nano -# subnet-id: subnet-0a63815aa47fb2deb -# security-group-id: sg-0633be23a5467c59c -# iam-role-name: runner_instance_profile # optional, requires additional permissions -# aws-resource-tags: > # optional, requires additional permissions -# [ -# {"Key": "Name", "Value": "ec2-github-runner"}, -# {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} -# ] # do-the-job: # name: Do the job on the runner -# needs: start-runner # required to start the main job when the runner is ready -# runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner +# runs-on: self-hosted # steps: +# - uses: actions/checkout@v2 # - name: Hello World -# run: echo 'Hello World!' -# stop-runner: -# name: Stop self-hosted EC2 runner -# needs: -# - start-runner # required to get output from the start-runner job -# - do-the-job # required to wait when the main job is done -# runs-on: ubuntu-latest -# if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs -# steps: -# - name: Configure AWS credentials -# uses: aws-actions/configure-aws-credentials@v1 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: ${{ secrets.AWS_REGION }} -# - name: Stop EC2 runner -# uses: machulav/ec2-github-runner@v2 -# with: -# mode: stop -# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} -# label: ${{ needs.start-runner.outputs.label }} -# ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file +# run: | +# echo 'Hello World!' +# sleep 15 +# echo 'Goodbye!' +#### +name: do-the-job +on: push +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-0e9085e60087ce171 + ec2-instance-type: t3.nano + subnet-id: subnet-0a63815aa47fb2deb + security-group-id: sg-0633be23a5467c59c + iam-role-name: runner_instance_profile # optional, requires additional permissions + aws-resource-tags: > # optional, requires additional permissions + [ + {"Key": "Name", "Value": "ec2-github-runner"}, + {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} + ] + do-the-job: + name: Do the job on the runner + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Hello World + run: echo 'Hello World!' + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner # required to get output from the start-runner job + - do-the-job # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file From 69617b6a92d211b4a53c915461515dcae8d47787 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Mon, 2 Dec 2024 01:53:07 +0100 Subject: [PATCH 012/148] Pipeline functional --- main.tf | 7 +++++-- modules/runner/iam.tf | 3 +-- modules/runner/main.tf | 10 ++++++---- modules/runner/network.tf | 26 -------------------------- modules/runner/userdata.sh | 20 +++++++++++--------- variables.tf | 2 +- 6 files changed, 24 insertions(+), 44 deletions(-) diff --git a/main.tf b/main.tf index fea9ae1..34f610c 100644 --- a/main.tf +++ b/main.tf @@ -8,6 +8,7 @@ module "vpc" { private_subnets = var.private_subnets public_subnets = var.public_subnets + map_public_ip_on_launch = true enable_nat_gateway = true default_vpc_tags = merge( @@ -20,10 +21,12 @@ module "vpc" { module "runner" { source = "./modules/runner" + vpc_id = module.vpc.vpc_id + subnet_id = tostring(module.vpc.public_subnets[0]) + ami_id = var.ami_id - vpc_id = module.vpc.default_vpc_id runner_instance_type = "t2.micro" - subnet_id = tostring(module.vpc.public_subnets[0]) + default_tags = merge( var.default_tags, { Name = "gh-runner" diff --git a/modules/runner/iam.tf b/modules/runner/iam.tf index ececd57..5917881 100644 --- a/modules/runner/iam.tf +++ b/modules/runner/iam.tf @@ -19,8 +19,7 @@ data "aws_iam_policy_document" "assume_role" { # Apply Least Privilege ASAP data "aws_iam_policy_document" "runner_policy" { statement { - # actions = ["ec2:ReplaceIamInstanceProfileAssociation", "ec2:AssociateIamInstanceProfile"] - actions = ["ec2:*"] + actions = ["ec2:ReplaceIamInstanceProfileAssociation", "ec2:AssociateIamInstanceProfile"] resources = ["*"] } statement { diff --git a/modules/runner/main.tf b/modules/runner/main.tf index 13a42a4..35b3a95 100644 --- a/modules/runner/main.tf +++ b/modules/runner/main.tf @@ -17,10 +17,7 @@ resource "aws_instance" "gh-runner" { subnet_id = var.subnet_id vpc_security_group_ids = [aws_security_group.gh-runner.id] - # network_interface { - # network_interface_id = aws_network_interface.gh-runner.id - # device_index = 0 - # } + key_name = aws_key_pair.runner.key_name tags = merge( var.default_tags, { @@ -33,3 +30,8 @@ resource "aws_instance" "gh-runner" { delete_on_termination = true } } + +resource "aws_key_pair" "runner" { + key_name = "deployer-key" + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDJaDTgLeCoQqvOD2Hbityz0WD+/jmLY7Wpy4Zmu81S2ejbSiU2AHAZil0KhCJzMDtoZfhbsntRCU1i5tIVnCyT1XOptMXrMn3h8LqVHjc7KqMZkOnPHFjUm/JBgAsxyM4NOVIgLykH4QRotRCBtMhjPWwDfpdgrlFciEmq6NEiyVkNRWT2RJ2FV9JqD15vs9i3Q/whmR6nbqb7o5HCPRz6s2wkQonsjP16v+MpPZjFswGMJxcsL4ZcKN4bvsElhwYVGDSS1R6Z4cn/CSU8bluRPIHWUSEZsW9vME7h32j2v79qBp5I8ACJbyQC2VstoHRWSOoVt/sQE3gLjGBd+goi7sQCHDVQnhstSPuxZOdEuxGDANSEyyo7TCiZrfRVZqcDtmUi1WmTkAzpvFjQYZT8hwIxsVbp2VG3tP6UwH3DH8ofxd6eIOvH27bxlbwzbOAkNG9/rwT4kGfyZZ2D8R9aH9PFXeeohiQkJegyRzzIWzHhxtL2v5i2Mxcbtnhj/kdzK0GUUymDjO3LK7+UW4kGEKCX/KxuuWWsrlrKPTMZu1x3nsDJD+gUgC33GOkY7zO0hSj4kXVxpPN+Q5RngNB9rHF7RPRMuS7TCF0V6ZfTRh9Q6DNDrGOrzlLmJj2yA0vB/V2rsLRA/TXVpTlE91/j/1vxsIFuZ99NspCUwbBABw== dilsilva.diego@gmail.com" +} \ No newline at end of file diff --git a/modules/runner/network.tf b/modules/runner/network.tf index 7a81ec1..0a1fec4 100644 --- a/modules/runner/network.tf +++ b/modules/runner/network.tf @@ -1,33 +1,7 @@ - -# resource "aws_network_interface" "gh-runner" { -# subnet_id = var.subnet_id - -# tags = merge( -# var.default_tags, { -# Name = "runner-primary_network_interface" -# }) -# } - - resource "aws_security_group" "gh-runner" { name = "gh-runner" description = "Allows necessary ports to gh runner communicate" vpc_id = var.vpc_id - -# ingress { -# from_port = 22 -# to_port = 22 -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# ipv6_cidr_blocks = ["::/0"] -# } -# egress { -# from_port = 0 -# to_port = 0 -# protocol = "-1" -# cidr_blocks = ["0.0.0.0/0"] -# ipv6_cidr_blocks = ["::/0"] -# } } resource "aws_vpc_security_group_ingress_rule" "allow_tls" { diff --git a/modules/runner/userdata.sh b/modules/runner/userdata.sh index 9e9745b..2ba5084 100644 --- a/modules/runner/userdata.sh +++ b/modules/runner/userdata.sh @@ -1,5 +1,12 @@ #!/bin/bash +#Github Actions +sudo yum update -y && \ +sudo yum install docker -y && \ +sudo yum install git -y && \ +sudo yum install libicu -y && \ +sudo systemctl enable docker + # Create a folder mkdir actions-runner && cd actions-runner # Download the latest runner package @@ -9,14 +16,9 @@ echo "ba46ba7ce3a4d7236b16fbe44419fb453bc08f866b24f04d549ec89f1722a29e actions- # Extract the installer tar xzf ./actions-runner-linux-x64-2.321.0.tar.gz # Create the runner and start the configuration experience -./config.sh --url https://github.com/dilsilva/surepay --token ADMDQVGMMFVDPL72MCABPZTHJNKUA +./config.sh --unattended --url https://github.com/dilsilva/surepay --token ADMDQVGIQVD3IP7QSANPRQLHJUHQE # Last step, run it! -./run.sh +./run.sh& - -#Github Actions -sudo yum update -y && \ -sudo yum install docker -y && \ -sudo yum install git -y && \ -sudo yum install libicu -y && \ -sudo systemctl enable docker \ No newline at end of file +# Setup as system service +./svc.sh \ No newline at end of file diff --git a/variables.tf b/variables.tf index 001a440..1577d4b 100644 --- a/variables.tf +++ b/variables.tf @@ -39,6 +39,6 @@ variable "azs" { #Runner variable "ami_id" { - default = "ami-02141377eee7defb9" + default = "ami-0e9085e60087ce171" type = string } From 8a3ec6e1b74d3e90f464dec788b84f1674fa90a4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 18:37:39 +0100 Subject: [PATCH 013/148] testing --- .github/workflows/runner.yaml | 208 +++++++++++++++++++++++----------- 1 file changed, 141 insertions(+), 67 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 6268233..ccc30b3 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,74 +1,148 @@ -# name: do-the-job -# on: push -# jobs: -# do-the-job: -# name: Do the job on the runner -# runs-on: self-hosted -# steps: -# - uses: actions/checkout@v2 -# - name: Hello World -# run: | -# echo 'Hello World!' -# sleep 15 -# echo 'Goodbye!' -#### name: do-the-job on: push jobs: - start-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-0e9085e60087ce171 - ec2-instance-type: t3.nano - subnet-id: subnet-0a63815aa47fb2deb - security-group-id: sg-0633be23a5467c59c - iam-role-name: runner_instance_profile # optional, requires additional permissions - aws-resource-tags: > # optional, requires additional permissions - [ - {"Key": "Name", "Value": "ec2-github-runner"}, - {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} - ] do-the-job: name: Do the job on the runner - needs: start-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + runs-on: self-hosted steps: + - uses: actions/checkout@v2 - name: Hello World - run: echo 'Hello World!' - stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - do-the-job # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file + run: | + echo 'Hello World!' + sleep 15 + echo 'Goodbye!' +# name: Deploy to ECR + +# on: + +# push: +# branches: [ master ] + +# env: +# ECR_REPOSITORY: flask-app +# EKS_CLUSTER_NAME: Kubernetes-demo +# AWS_REGION: us-east-1 + +# jobs: + +# build: + +# name: Deployment +# runs-on: self-hosted + +# steps: + +# - name: Set short git commit SHA +# id: commit +# uses: prompt/actions-commit-hash@v2 + +# # Checkout and test app code +# - name: Check out code +# uses: actions/checkout@v2 + +# with: dilsilva/go-load-generator +# - name: Setup Go +# uses: actions/setup-go@v5 +# with: +# go-version: '1.23.x' +# - name: Install dependencies +# run: go get ./cmd/loadgen/ +# - name: Test with the Go CLI +# run: go test -v -cover ./... + + +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: ${{env.AWS_REGION}} + +# - name: Login to Amazon ECR +# id: login-ecr +# uses: aws-actions/amazon-ecr-login@v1 + +# - name: Build, tag, and push image to Amazon ECR +# env: +# ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} +# IMAGE_TAG: ${{ steps.commit.outputs.short }} +# run: | +# docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . +# docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + +# #Insert vlnerability scan after build + +# - name: Update kube config +# run: aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION + +# #Insert Sign Image verification step before deployment + +# - name: Deploy to EKS +# env: +# ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} +# IMAGE_TAG: ${{ steps.commit.outputs.short }} +# run: | +# sed -i.bak "s|DOCKER_IMAGE|$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG|g" manifests/hello-app-deployment.yaml && \ +# kubectl apply -f manifests/hello-app-deployment.yaml +# kubectl apply -f manifests/hello-app-service.yaml +#### +# name: do-the-job +# on: push +# jobs: +# start-runner: +# name: Start self-hosted EC2 runner +# runs-on: ubuntu-latest +# outputs: +# label: ${{ steps.start-ec2-runner.outputs.label }} +# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} +# steps: +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: ${{ secrets.AWS_REGION }} +# - name: Start EC2 runner +# id: start-ec2-runner +# uses: machulav/ec2-github-runner@v2 +# with: +# mode: start +# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} +# ec2-image-id: ami-0e9085e60087ce171 +# ec2-instance-type: t3.nano +# subnet-id: subnet-0a63815aa47fb2deb +# security-group-id: sg-0633be23a5467c59c +# iam-role-name: runner_instance_profile # optional, requires additional permissions +# aws-resource-tags: > # optional, requires additional permissions +# [ +# {"Key": "Name", "Value": "ec2-github-runner"}, +# {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} +# ] +# do-the-job: +# name: Do the job on the runner +# needs: start-runner # required to start the main job when the runner is ready +# runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner +# steps: +# - name: Hello World +# run: echo 'Hello World!' +# stop-runner: +# name: Stop self-hosted EC2 runner +# needs: +# - start-runner # required to get output from the start-runner job +# - do-the-job # required to wait when the main job is done +# runs-on: ubuntu-latest +# if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs +# steps: +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: ${{ secrets.AWS_REGION }} +# - name: Stop EC2 runner +# uses: machulav/ec2-github-runner@v2 +# with: +# mode: stop +# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} +# label: ${{ needs.start-runner.outputs.label }} +# ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file From 6424fe290ba1fa29418875930566467a2ba46fa1 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:17:50 +0100 Subject: [PATCH 014/148] testing --- .github/workflows/runner.yaml | 206 ++++++++++++---------------------- 1 file changed, 73 insertions(+), 133 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index ccc30b3..d57f7c1 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,148 +1,88 @@ -name: do-the-job -on: push -jobs: - do-the-job: - name: Do the job on the runner - runs-on: self-hosted - steps: - - uses: actions/checkout@v2 - - name: Hello World - run: | - echo 'Hello World!' - sleep 15 - echo 'Goodbye!' -# name: Deploy to ECR - -# on: - -# push: -# branches: [ master ] - -# env: -# ECR_REPOSITORY: flask-app -# EKS_CLUSTER_NAME: Kubernetes-demo -# AWS_REGION: us-east-1 - +# name: do-the-job +# on: push # jobs: - -# build: - -# name: Deployment +# do-the-job: +# name: Do the job on the runner # runs-on: self-hosted - # steps: +# - uses: actions/checkout@v2 +# - name: Hello World +# run: | +# echo 'Hello World!' +# sleep 15 +# echo 'Goodbye!' +name: Deploy to ECR -# - name: Set short git commit SHA -# id: commit -# uses: prompt/actions-commit-hash@v2 +on: push +env: + ECR_REPOSITORY: go-app + EKS_CLUSTER_NAME: Kubernetes-demo + AWS_REGION: eu-west-1 +jobs: + build: + name: Deployment + runs-on: self-hosted + steps: + - name: Set short git commit SHA + id: commit + uses: prompt/actions-commit-hash@v2 -# # Checkout and test app code -# - name: Check out code -# uses: actions/checkout@v2 + # Checkout and test app code + - name: Check out code + uses: actions/checkout@v2 -# with: dilsilva/go-load-generator -# - name: Setup Go -# uses: actions/setup-go@v5 -# with: -# go-version: '1.23.x' -# - name: Install dependencies -# run: go get ./cmd/loadgen/ -# - name: Test with the Go CLI -# run: go test -v -cover ./... + with: dilsilva/go-load-generator + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23.x' + - name: Install dependencies + run: go get ./cmd/loadgen/ + - name: Test with the Go CLI + run: go test -v -cover ./... -# - name: Configure AWS credentials -# uses: aws-actions/configure-aws-credentials@v1 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: ${{env.AWS_REGION}} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{env.AWS_REGION}} -# - name: Login to Amazon ECR -# id: login-ecr -# uses: aws-actions/amazon-ecr-login@v1 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 -# - name: Build, tag, and push image to Amazon ECR -# env: -# ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} -# IMAGE_TAG: ${{ steps.commit.outputs.short }} -# run: | -# docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . -# docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + - name: Build, tag, and push image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ steps.commit.outputs.short }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -# #Insert vlnerability scan after build + #Insert vulnerability scan after build + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ${{ steps.login-ecr.outputs.registry }} + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' -# - name: Update kube config -# run: aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION + # - name: Update kube config + # run: aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION -# #Insert Sign Image verification step before deployment + # #Insert Sign Image verification step before deployment -# - name: Deploy to EKS -# env: -# ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} -# IMAGE_TAG: ${{ steps.commit.outputs.short }} -# run: | -# sed -i.bak "s|DOCKER_IMAGE|$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG|g" manifests/hello-app-deployment.yaml && \ -# kubectl apply -f manifests/hello-app-deployment.yaml -# kubectl apply -f manifests/hello-app-service.yaml + # - name: Deploy to EKS + # env: + # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # IMAGE_TAG: ${{ steps.commit.outputs.short }} + # run: | + # sed -i.bak "s|DOCKER_IMAGE|$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG|g" manifests/hello-app-deployment.yaml && \ + # kubectl apply -f manifests/hello-app-deployment.yaml + # kubectl apply -f manifests/hello-app-service.yaml #### -# name: do-the-job -# on: push -# jobs: -# start-runner: -# name: Start self-hosted EC2 runner -# runs-on: ubuntu-latest -# outputs: -# label: ${{ steps.start-ec2-runner.outputs.label }} -# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} -# steps: -# - name: Configure AWS credentials -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: ${{ secrets.AWS_REGION }} -# - name: Start EC2 runner -# id: start-ec2-runner -# uses: machulav/ec2-github-runner@v2 -# with: -# mode: start -# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} -# ec2-image-id: ami-0e9085e60087ce171 -# ec2-instance-type: t3.nano -# subnet-id: subnet-0a63815aa47fb2deb -# security-group-id: sg-0633be23a5467c59c -# iam-role-name: runner_instance_profile # optional, requires additional permissions -# aws-resource-tags: > # optional, requires additional permissions -# [ -# {"Key": "Name", "Value": "ec2-github-runner"}, -# {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} -# ] -# do-the-job: -# name: Do the job on the runner -# needs: start-runner # required to start the main job when the runner is ready -# runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner -# steps: -# - name: Hello World -# run: echo 'Hello World!' -# stop-runner: -# name: Stop self-hosted EC2 runner -# needs: -# - start-runner # required to get output from the start-runner job -# - do-the-job # required to wait when the main job is done -# runs-on: ubuntu-latest -# if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs -# steps: -# - name: Configure AWS credentials -# uses: aws-actions/configure-aws-credentials@v1 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: ${{ secrets.AWS_REGION }} -# - name: Stop EC2 runner -# uses: machulav/ec2-github-runner@v2 -# with: -# mode: stop -# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} -# label: ${{ needs.start-runner.outputs.label }} -# ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file From 5a72b5e14044d92b63a5c7a92a8259ceda9c465f Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:22:46 +0100 Subject: [PATCH 015/148] testing --- .gitignore | 3 +- main.tf | 98 +++++++++++++++++++++++++++++++----- modules/keypair/main.tf | 4 ++ modules/keypair/outputs.tf | 9 ++++ modules/keypair/variables.tf | 1 + modules/runner/main.tf | 8 +-- modules/runner/network.tf | 4 +- modules/runner/variables.tf | 3 +- variables.tf | 18 +++++-- 9 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 modules/keypair/main.tf create mode 100644 modules/keypair/outputs.tf create mode 100644 modules/keypair/variables.tf diff --git a/.gitignore b/.gitignore index 779c0d6..17e11f2 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ override.tf.json terraform.rc # Lock file -.terraform.lock.hcl \ No newline at end of file +.terraform.lock.hcl +NOTES \ No newline at end of file diff --git a/main.tf b/main.tf index 34f610c..9c11afa 100644 --- a/main.tf +++ b/main.tf @@ -8,28 +8,100 @@ module "vpc" { private_subnets = var.private_subnets public_subnets = var.public_subnets - map_public_ip_on_launch = true + #Single NAT Gateway https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/v5.16.0?tab=readme-ov-file#single-nat-gateway enable_nat_gateway = true + single_nat_gateway = true + + create_database_subnet_group = false + create_elasticache_subnet_group = false + create_redshift_subnet_group = false + manage_default_network_acl = false + manage_default_route_table = false + manage_default_security_group = false + + default_vpc_tags = merge( var.default_tags, { - Name = "SurepayVPC" + Name = "vpc" }, ) } -module "runner" { - source = "./modules/runner" +# module "runner" { +# source = "./modules/runner" + +# vpc_id = module.vpc.vpc_id +# subnet_id = tostring(module.vpc.public_subnets[0]) +# key_pair_name = module.keypair.key_pair_name + +# ami_id = var.ami_id +# runner_instance_type = "t2.micro" + +# default_tags = merge( +# var.default_tags, { +# Name = "gh-runner" +# }, +# ) +# } + +# module "eks" { +# source = "terraform-aws-modules/eks/aws" +# version = "~> 20.0" + +# cluster_name = "${var.project}-al2023" +# cluster_version = "1.31" - vpc_id = module.vpc.vpc_id - subnet_id = tostring(module.vpc.public_subnets[0]) - - ami_id = var.ami_id - runner_instance_type = "t2.micro" +# # EKS Addons +# cluster_addons = { +# coredns = {} +# eks-pod-identity-agent = {} +# kube-proxy = {} +# vpc-cni = {} +# } - default_tags = merge( +# vpc_id = module.vpc.vpc_id +# subnet_ids = module.vpc.public_subnets + +# eks_managed_node_groups = { +# main = { +# # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups +# instance_types = [var.instance_type] + +# min_size = 1 +# max_size = 3 +# # This value is ignored after the initial creation +# # https://github.com/bryantbiggs/eks-desired-size-hack +# desired_size = 1 +# } +# } + +# tags = merge( +# var.default_tags, { +# Name = "eks" +# }, +# ) +# } + +module "bastion" { + source = "Guimove/bastion/aws" + bastion_ami = "ami-0e9085e60087ce171" + bucket_name = "${var.project}-bastion-bucket" + region = var.region + vpc_id = module.vpc.vpc_id + is_lb_private = "false" + bastion_host_key_pair = module.keypair.key_pair_name + create_dns_record = "false" + bastion_iam_policy_name = "${var.project}BastionHostPolicy" + elb_subnets = [module.vpc.public_subnets[0], module.vpc.public_subnets[1]] + auto_scaling_group_subnets = [module.vpc.public_subnets[0], module.vpc.public_subnets[1]] + tags = merge( var.default_tags, { - Name = "gh-runner" - }, - ) + "name" = "${var.project}-bastion" + }, ) +} + +module "keypair" { + source = "./modules/keypair" + project = var.project } \ No newline at end of file diff --git a/modules/keypair/main.tf b/modules/keypair/main.tf new file mode 100644 index 0000000..40b957c --- /dev/null +++ b/modules/keypair/main.tf @@ -0,0 +1,4 @@ +resource "aws_key_pair" "keypair" { + key_name = "${var.project}-keypair" + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDJaDTgLeCoQqvOD2Hbityz0WD+/jmLY7Wpy4Zmu81S2ejbSiU2AHAZil0KhCJzMDtoZfhbsntRCU1i5tIVnCyT1XOptMXrMn3h8LqVHjc7KqMZkOnPHFjUm/JBgAsxyM4NOVIgLykH4QRotRCBtMhjPWwDfpdgrlFciEmq6NEiyVkNRWT2RJ2FV9JqD15vs9i3Q/whmR6nbqb7o5HCPRz6s2wkQonsjP16v+MpPZjFswGMJxcsL4ZcKN4bvsElhwYVGDSS1R6Z4cn/CSU8bluRPIHWUSEZsW9vME7h32j2v79qBp5I8ACJbyQC2VstoHRWSOoVt/sQE3gLjGBd+goi7sQCHDVQnhstSPuxZOdEuxGDANSEyyo7TCiZrfRVZqcDtmUi1WmTkAzpvFjQYZT8hwIxsVbp2VG3tP6UwH3DH8ofxd6eIOvH27bxlbwzbOAkNG9/rwT4kGfyZZ2D8R9aH9PFXeeohiQkJegyRzzIWzHhxtL2v5i2Mxcbtnhj/kdzK0GUUymDjO3LK7+UW4kGEKCX/KxuuWWsrlrKPTMZu1x3nsDJD+gUgC33GOkY7zO0hSj4kXVxpPN+Q5RngNB9rHF7RPRMuS7TCF0V6ZfTRh9Q6DNDrGOrzlLmJj2yA0vB/V2rsLRA/TXVpTlE91/j/1vxsIFuZ99NspCUwbBABw== dilsilva.diego@gmail.com" +} \ No newline at end of file diff --git a/modules/keypair/outputs.tf b/modules/keypair/outputs.tf new file mode 100644 index 0000000..bde488d --- /dev/null +++ b/modules/keypair/outputs.tf @@ -0,0 +1,9 @@ +output "key_pair_name" { + description = "The name of the key for the EC2" + value = aws_key_pair.keypair.key_name +} + +output "key_pair_arn" { + description = "The ARN of the key for the EC2" + value = aws_key_pair.keypair.arn +} \ No newline at end of file diff --git a/modules/keypair/variables.tf b/modules/keypair/variables.tf new file mode 100644 index 0000000..eeec148 --- /dev/null +++ b/modules/keypair/variables.tf @@ -0,0 +1 @@ +variable "project" {} diff --git a/modules/runner/main.tf b/modules/runner/main.tf index 35b3a95..bd3f420 100644 --- a/modules/runner/main.tf +++ b/modules/runner/main.tf @@ -14,10 +14,10 @@ resource "aws_instance" "gh-runner" { iam_instance_profile = aws_iam_instance_profile.runner_instance_profile.name user_data = file("${path.module}/userdata.sh") - subnet_id = var.subnet_id - vpc_security_group_ids = [aws_security_group.gh-runner.id] + subnet_id = var.subnet_id + vpc_security_group_ids = [aws_security_group.gh-runner.id] - key_name = aws_key_pair.runner.key_name + key_name = var.key_pair_name tags = merge( var.default_tags, { @@ -31,7 +31,7 @@ resource "aws_instance" "gh-runner" { } } -resource "aws_key_pair" "runner" { +resource "aws_key_pair" "surepay-keypair" { key_name = "deployer-key" public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDJaDTgLeCoQqvOD2Hbityz0WD+/jmLY7Wpy4Zmu81S2ejbSiU2AHAZil0KhCJzMDtoZfhbsntRCU1i5tIVnCyT1XOptMXrMn3h8LqVHjc7KqMZkOnPHFjUm/JBgAsxyM4NOVIgLykH4QRotRCBtMhjPWwDfpdgrlFciEmq6NEiyVkNRWT2RJ2FV9JqD15vs9i3Q/whmR6nbqb7o5HCPRz6s2wkQonsjP16v+MpPZjFswGMJxcsL4ZcKN4bvsElhwYVGDSS1R6Z4cn/CSU8bluRPIHWUSEZsW9vME7h32j2v79qBp5I8ACJbyQC2VstoHRWSOoVt/sQE3gLjGBd+goi7sQCHDVQnhstSPuxZOdEuxGDANSEyyo7TCiZrfRVZqcDtmUi1WmTkAzpvFjQYZT8hwIxsVbp2VG3tP6UwH3DH8ofxd6eIOvH27bxlbwzbOAkNG9/rwT4kGfyZZ2D8R9aH9PFXeeohiQkJegyRzzIWzHhxtL2v5i2Mxcbtnhj/kdzK0GUUymDjO3LK7+UW4kGEKCX/KxuuWWsrlrKPTMZu1x3nsDJD+gUgC33GOkY7zO0hSj4kXVxpPN+Q5RngNB9rHF7RPRMuS7TCF0V6ZfTRh9Q6DNDrGOrzlLmJj2yA0vB/V2rsLRA/TXVpTlE91/j/1vxsIFuZ99NspCUwbBABw== dilsilva.diego@gmail.com" } \ No newline at end of file diff --git a/modules/runner/network.tf b/modules/runner/network.tf index 0a1fec4..72d03ec 100644 --- a/modules/runner/network.tf +++ b/modules/runner/network.tf @@ -1,7 +1,7 @@ resource "aws_security_group" "gh-runner" { - name = "gh-runner" + name = "gh-runner" description = "Allows necessary ports to gh runner communicate" - vpc_id = var.vpc_id + vpc_id = var.vpc_id } resource "aws_vpc_security_group_ingress_rule" "allow_tls" { diff --git a/modules/runner/variables.tf b/modules/runner/variables.tf index b34a2e1..b8cf7e7 100644 --- a/modules/runner/variables.tf +++ b/modules/runner/variables.tf @@ -2,4 +2,5 @@ variable "runner_instance_type" {} variable "default_tags" {} variable "subnet_id" {} variable "ami_id" {} -variable "vpc_id" {} \ No newline at end of file +variable "vpc_id" {} +variable "key_pair_name" {} \ No newline at end of file diff --git a/variables.tf b/variables.tf index 1577d4b..017164d 100644 --- a/variables.tf +++ b/variables.tf @@ -1,3 +1,5 @@ +#Global + variable "project" { default = "surepay" } @@ -18,27 +20,35 @@ variable "default_tags" { type = map(string) } +#VPC variable "vpc_cidr" { default = "10.0.0.0/16" type = string } variable "private_subnets" { - default = ["10.0.1.0/24"] + default = ["10.0.1.0/24", "10.0.2.0/24"] type = list(string) } + variable "public_subnets" { - default = ["10.0.101.0/24"] + default = ["10.0.101.0/24", "10.0.102.0/24"] type = list(string) } + variable "azs" { - default = ["eu-west-1a"] + default = ["eu-west-1a", "eu-west-1b"] type = list(string) -} +} #Runner variable "ami_id" { default = "ami-0e9085e60087ce171" type = string } + +#EKS +variable "instance_type" { + default = "t2.micro" +} \ No newline at end of file From c8aedc5d5a4358bf6247949951a7df98b358d947 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:23:28 +0100 Subject: [PATCH 016/148] testing --- .github/workflows/runner.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index d57f7c1..bc55bab 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -30,8 +30,9 @@ jobs: # Checkout and test app code - name: Check out code uses: actions/checkout@v2 + with: + repository: 'https://github.com/dilsilva/go-load-generator/' - with: dilsilva/go-load-generator - name: Setup Go uses: actions/setup-go@v5 with: From f4832acc3540f0ddd14577d40492160a4b39c5d2 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:23:57 +0100 Subject: [PATCH 017/148] testing --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index bc55bab..db2cfd8 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -31,7 +31,7 @@ jobs: - name: Check out code uses: actions/checkout@v2 with: - repository: 'https://github.com/dilsilva/go-load-generator/' + repository: 'dilsilva/go-load-generator/' - name: Setup Go uses: actions/setup-go@v5 From 69b67220609e0861f7613357306a53386678eef8 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:24:29 +0100 Subject: [PATCH 018/148] testing --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index db2cfd8..e49f3c0 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -31,7 +31,7 @@ jobs: - name: Check out code uses: actions/checkout@v2 with: - repository: 'dilsilva/go-load-generator/' + repository: 'dilsilva/go-load-generator' - name: Setup Go uses: actions/setup-go@v5 From 4a70565fe9547aa0c076e73d065579dad19582ab Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:37:52 +0100 Subject: [PATCH 019/148] testing --- .github/workflows/runner.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index e49f3c0..6233978 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -20,7 +20,7 @@ env: AWS_REGION: eu-west-1 jobs: build: - name: Deployment + name: CI runs-on: self-hosted steps: - name: Set short git commit SHA @@ -39,10 +39,10 @@ jobs: go-version: '1.23.x' - name: Install dependencies run: go get ./cmd/loadgen/ - - name: Test with the Go CLI - run: go test -v -cover ./... - + # - name: Unit Test + # run: go test -v -cover ./... + name: CD - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From dc3cae92b9257cb3d119dac6a114e1ac2fd718f4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:39:02 +0100 Subject: [PATCH 020/148] testing --- .github/workflows/runner.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 6233978..7a7f8e1 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -33,12 +33,12 @@ jobs: with: repository: 'dilsilva/go-load-generator' - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23.x' - - name: Install dependencies - run: go get ./cmd/loadgen/ + # - name: Setup Go + # uses: actions/setup-go@v5 + # with: + # go-version: '1.23.x' + # - name: Install dependencies + # run: go get ./cmd/loadgen/ # - name: Unit Test # run: go test -v -cover ./... From d6e8c01fc15eaa8d35f44a9b2806d56d47988418 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:39:36 +0100 Subject: [PATCH 021/148] testing --- .github/workflows/runner.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 7a7f8e1..5dcf54c 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -33,16 +33,15 @@ jobs: with: repository: 'dilsilva/go-load-generator' - # - name: Setup Go - # uses: actions/setup-go@v5 - # with: - # go-version: '1.23.x' - # - name: Install dependencies - # run: go get ./cmd/loadgen/ + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23.x' + - name: Install dependencies + run: go get ./cmd/loadgen/ # - name: Unit Test # run: go test -v -cover ./... - name: CD - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From b32526f2ef494779ffbec840d520b0ac4552921d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:49:46 +0100 Subject: [PATCH 022/148] testing --- main.tf | 1 + variables.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 9c11afa..7aba5e5 100644 --- a/main.tf +++ b/main.tf @@ -89,6 +89,7 @@ module "bastion" { bucket_name = "${var.project}-bastion-bucket" region = var.region vpc_id = module.vpc.vpc_id + instance_type = var.instance_type is_lb_private = "false" bastion_host_key_pair = module.keypair.key_pair_name create_dns_record = "false" diff --git a/variables.tf b/variables.tf index 017164d..83a1346 100644 --- a/variables.tf +++ b/variables.tf @@ -50,5 +50,5 @@ variable "ami_id" { #EKS variable "instance_type" { - default = "t2.micro" + default = "t2.medium" } \ No newline at end of file From be4fe052b6a1569a9cb943aae9a5c3d30d578660 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:51:22 +0100 Subject: [PATCH 023/148] testing --- .github/workflows/runner.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5dcf54c..b64afa7 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -39,8 +39,9 @@ jobs: go-version: '1.23.x' - name: Install dependencies run: go get ./cmd/loadgen/ - # - name: Unit Test - # run: go test -v -cover ./... + - name: Unit Test + run: go test -v -cover ./... + run: ls -la - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 From 5e69e9ad7511fe9c9f20826c445f4cdcf442defa Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:51:54 +0100 Subject: [PATCH 024/148] testing --- .github/workflows/runner.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index b64afa7..0d8e0ee 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -40,8 +40,7 @@ jobs: - name: Install dependencies run: go get ./cmd/loadgen/ - name: Unit Test - run: go test -v -cover ./... - run: ls -la + run: go test -v -cover ./... && ls -la - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 From a547065b940f39ae68c4b7764879de9b43361bab Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:53:52 +0100 Subject: [PATCH 025/148] testing --- .github/workflows/runner.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 0d8e0ee..ebd234e 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -38,9 +38,9 @@ jobs: with: go-version: '1.23.x' - name: Install dependencies - run: go get ./cmd/loadgen/ - - name: Unit Test - run: go test -v -cover ./... && ls -la + run: go get ./cmd/loadgen/ && ls -la + # - name: Unit Test + # run: go test -v -cover ./... - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -58,8 +58,8 @@ jobs: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ steps.commit.outputs.short }} run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . + sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG #Insert vulnerability scan after build - name: Run Trivy vulnerability scanner From bcbd0c0db38d34630cd80291020eedfa188f7e87 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 20:59:33 +0100 Subject: [PATCH 026/148] testing --- main.tf | 2 +- variables.tf | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 7aba5e5..71a05fe 100644 --- a/main.tf +++ b/main.tf @@ -85,7 +85,7 @@ module "vpc" { module "bastion" { source = "Guimove/bastion/aws" - bastion_ami = "ami-0e9085e60087ce171" + bastion_ami = var.ami_id bucket_name = "${var.project}-bastion-bucket" region = var.region vpc_id = module.vpc.vpc_id diff --git a/variables.tf b/variables.tf index 83a1346..dcfdca9 100644 --- a/variables.tf +++ b/variables.tf @@ -44,7 +44,8 @@ variable "azs" { #Runner variable "ami_id" { - default = "ami-0e9085e60087ce171" + default = "ami-0e9085e60087ce171" #Ubuntu + # default = "ami-047bb4163c506cd98" #Anazon Linyx type = string } From 3d68cf2db4eec7de0d2f513e5c0f03c18cf1d49a Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Tue, 3 Dec 2024 21:12:29 +0100 Subject: [PATCH 027/148] testing --- .github/workflows/runner.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index ebd234e..5404ab4 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -51,11 +51,16 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: "true" + registry-type: public + skip-logout: false - name: Build, tag, and push image to Amazon ECR env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: surepay IMAGE_TAG: ${{ steps.commit.outputs.short }} run: | sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . From c02bef326b0766477f3797b20ad1159bd66c24f0 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 15:59:13 +0100 Subject: [PATCH 028/148] testing pipeline --- .github/workflows/runner.yaml | 47 ++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5404ab4..279b900 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -41,30 +41,27 @@ jobs: run: go get ./cmd/loadgen/ && ls -la # - name: Unit Test # run: go test -v -cover ./... - - - name: Configure AWS credentials + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{env.AWS_REGION}} + aws-region: us-east-2 - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - with: - mask-password: "true" - registry-type: public - skip-logout: false + uses: aws-actions/amazon-ecr-login@v1 - name: Build, tag, and push image to Amazon ECR + id: build-image env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: surepay - IMAGE_TAG: ${{ steps.commit.outputs.short }} + ECR_REPOSITORY: my-ecr-repo + IMAGE_TAG: ${{ github.sha }} run: | - sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . - sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT #Insert vulnerability scan after build - name: Run Trivy vulnerability scanner @@ -81,13 +78,23 @@ jobs: # run: aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION # #Insert Sign Image verification step before deployment + - name: Download task definition + run: | + aws ecs describe-task-definition --task-definition my-task-definition-family --query taskDefinition > task-definition.json + + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: my-container + image: ${{ steps.build-image.outputs.image }} - # - name: Deploy to EKS - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # IMAGE_TAG: ${{ steps.commit.outputs.short }} - # run: | - # sed -i.bak "s|DOCKER_IMAGE|$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG|g" manifests/hello-app-deployment.yaml && \ - # kubectl apply -f manifests/hello-app-deployment.yaml - # kubectl apply -f manifests/hello-app-service.yaml + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: my-service + cluster: my-cluster + wait-for-service-stability: true #### From a9f751542f36031f7a70f3842df3201631bb7b0d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:00:31 +0100 Subject: [PATCH 029/148] testing pipeline --- .github/workflows/runner.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 279b900..239c64c 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -39,9 +39,9 @@ jobs: go-version: '1.23.x' - name: Install dependencies run: go get ./cmd/loadgen/ && ls -la - # - name: Unit Test - # run: go test -v -cover ./... - - name: Configure AWS credentials + - name: Unit Test + run: go test -v -cover ./... + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -81,7 +81,7 @@ jobs: - name: Download task definition run: | aws ecs describe-task-definition --task-definition my-task-definition-family --query taskDefinition > task-definition.json - + - name: Fill in the new image ID in the Amazon ECS task definition id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 From 944657bd688abc9b2bb7de9da363de80e9974f27 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:05:34 +0100 Subject: [PATCH 030/148] testing pipeline --- .github/workflows/runner.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 239c64c..e2f6b96 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -23,9 +23,6 @@ jobs: name: CI runs-on: self-hosted steps: - - name: Set short git commit SHA - id: commit - uses: prompt/actions-commit-hash@v2 # Checkout and test app code - name: Check out code From bec5a67b8a9cba7eaee63f4c86133e14d4557842 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:30:15 +0100 Subject: [PATCH 031/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index e2f6b96..d7120e5 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -56,8 +56,8 @@ jobs: ECR_REPOSITORY: my-ecr-repo IMAGE_TAG: ${{ github.sha }} run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT #Insert vulnerability scan after build From 664b5dcc95b62064d8035e203d05ec83c06721c2 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:46:57 +0100 Subject: [PATCH 032/148] testing pipeline --- .github/workflows/runner.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index d7120e5..2dfc1d5 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -15,9 +15,8 @@ name: Deploy to ECR on: push env: - ECR_REPOSITORY: go-app - EKS_CLUSTER_NAME: Kubernetes-demo AWS_REGION: eu-west-1 + AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} jobs: build: name: CI @@ -47,13 +46,15 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + run: | + # Get the ECR login password and use it to log in to the ECR registry + result=$(aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.your-region.amazonaws.com) - name: Build, tag, and push image to Amazon ECR id: build-image env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: my-ecr-repo + ECR_REPOSITORY: go-app IMAGE_TAG: ${{ github.sha }} run: | sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . From 12b56220294de10c69644ce75994570a28692447 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:50:00 +0100 Subject: [PATCH 033/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 2dfc1d5..dd04384 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -48,7 +48,7 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.your-region.amazonaws.com) + result=$(aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR id: build-image From 70ed4c3d9949b5af2d61d8a8177c9eb20107b1fd Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:52:05 +0100 Subject: [PATCH 034/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index dd04384..ec21f35 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -53,7 +53,7 @@ jobs: - name: Build, tag, and push image to Amazon ECR id: build-image env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REGISTRY: $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com ECR_REPOSITORY: go-app IMAGE_TAG: ${{ github.sha }} run: | From b3aca4498fd89f23ed85d55810037a3b589f52b6 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:55:12 +0100 Subject: [PATCH 035/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index ec21f35..929f7f5 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -53,7 +53,7 @@ jobs: - name: Build, tag, and push image to Amazon ECR id: build-image env: - ECR_REGISTRY: $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} ECR_REPOSITORY: go-app IMAGE_TAG: ${{ github.sha }} run: | From d94334645289c2ac949a04ff1a6630181401df28 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 16:58:50 +0100 Subject: [PATCH 036/148] testing pipeline --- .github/workflows/runner.yaml | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 929f7f5..9185061 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -46,20 +46,30 @@ jobs: - name: Login to Amazon ECR id: login-ecr - run: | - # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + uses: aws-actions/amazon-ecr-login@v2 - - name: Build, tag, and push image to Amazon ECR - id: build-image + - name: Build, tag, and push docker image to Amazon ECR env: - ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} - ECR_REPOSITORY: go-app + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: go-app IMAGE_TAG: ${{ github.sha }} run: | - sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + sudo docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . + sudo docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG + + # - name: Login to Amazon ECR + # id: login-ecr + # run: | + # # Get the ECR login password and use it to log in to the ECR registry + # aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + + # - name: Build, tag, and push image to Amazon ECR + # id: build-image + + # run: | + # sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + # sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + # echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT #Insert vulnerability scan after build - name: Run Trivy vulnerability scanner From bcad2e3155575c98f3621e47591c251d3bf4c214 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:07:10 +0100 Subject: [PATCH 037/148] testing pipeline --- .github/workflows/runner.yaml | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 9185061..6da7376 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -44,32 +44,31 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + run: | + # Get the ECR login password and use it to log in to the ECR registry + aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - - name: Build, tag, and push docker image to Amazon ECR + - name: Build, tag, and push image to Amazon ECR env: - REGISTRY: ${{ steps.login-ecr.outputs.registry }} - REPOSITORY: go-app + ECR_REGISTRY: {{ secrets.ECR_REGISTRY }} + ECR_REPOSITORY: go-app + TAG_COMMIT: your-tag-commit IMAGE_TAG: ${{ github.sha }} run: | - sudo docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . - sudo docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG + # Build the Docker image using the specified tags + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + + # Push the Docker image to the ECR repository using Docker config from the workspace + sudo docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - # - name: Login to Amazon ECR - # id: login-ecr - # run: | - # # Get the ECR login password and use it to log in to the ECR registry - # aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + # Remove the Docker config file after pushing the image + rm -fr ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID}/config.json - # - name: Build, tag, and push image to Amazon ECR - # id: build-image + $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT >> $GITHUB_OUTPUT - # run: | - # sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - # sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - # echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT #Insert vulnerability scan after build - name: Run Trivy vulnerability scanner @@ -82,8 +81,6 @@ jobs: vuln-type: 'os,library' severity: 'CRITICAL,HIGH' - # - name: Update kube config - # run: aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION # #Insert Sign Image verification step before deployment - name: Download task definition From 3e68406848b6d6d947939001dfc349e5efb2b2f4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:08:44 +0100 Subject: [PATCH 038/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 6da7376..4e88d32 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -53,7 +53,7 @@ jobs: - name: Build, tag, and push image to Amazon ECR env: - ECR_REGISTRY: {{ secrets.ECR_REGISTRY }} + ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} ECR_REPOSITORY: go-app TAG_COMMIT: your-tag-commit IMAGE_TAG: ${{ github.sha }} From e1348d0fc44eadc45ecf1016bbe34a08f25c88dc Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:11:06 +0100 Subject: [PATCH 039/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 4e88d32..39e5b99 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -49,7 +49,7 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - name: Build, tag, and push image to Amazon ECR env: From e3398bd9351c0b87ee008191f20979bf2d2bc8df Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:11:49 +0100 Subject: [PATCH 040/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 39e5b99..5d9414f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -59,7 +59,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} run: | # Build the Docker image using the specified tags - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . # Push the Docker image to the ECR repository using Docker config from the workspace sudo docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT From 9084ea2388e7308a85c0a4902090904956212128 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:13:21 +0100 Subject: [PATCH 041/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5d9414f..7c2ca9a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -49,8 +49,8 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - + result=$(aws ecr get-login-password --region your-region | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin xxxxxxxxx.dkr.ecr.your-region.amazonaws.com) + - name: Build, tag, and push image to Amazon ECR env: ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} From d096f1b54750589a85b56a75047ce17f97e0b221 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:14:45 +0100 Subject: [PATCH 042/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 7c2ca9a..ef52db6 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -49,7 +49,7 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region your-region | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin xxxxxxxxx.dkr.ecr.your-region.amazonaws.com) + result=$(aws ecr get-login-password --region your-region | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: From d458b1e2c2e77453e89c0f5449b1976c76054723 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:15:26 +0100 Subject: [PATCH 043/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index ef52db6..deced77 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -49,7 +49,7 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region your-region | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + result=$(aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: From 9e73aba937245cd26149c0907b1e00cbf41860a0 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:19:01 +0100 Subject: [PATCH 044/148] testing pipeline --- .github/workflows/runner.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index deced77..645d234 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -49,14 +49,13 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} ECR_REPOSITORY: go-app - TAG_COMMIT: your-tag-commit - IMAGE_TAG: ${{ github.sha }} + TAG_COMMIT: ${{ github.sha }} run: | # Build the Docker image using the specified tags sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . @@ -65,7 +64,7 @@ jobs: sudo docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT # Remove the Docker config file after pushing the image - rm -fr ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID}/config.json + rm -fr ~/.config.json $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT >> $GITHUB_OUTPUT From f4fd102b4b189fd6376aa271cb83971089308476 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:21:30 +0100 Subject: [PATCH 045/148] testing pipeline --- .github/workflows/runner.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 645d234..4cf9b63 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -45,11 +45,11 @@ jobs: aws-region: us-east-2 - - name: Login to Amazon ECR - id: login-ecr - run: | - # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + # - name: Login to Amazon ECR + # id: login-ecr + # run: | + # # Get the ECR login password and use it to log in to the ECR registry + # result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: @@ -57,6 +57,9 @@ jobs: ECR_REPOSITORY: go-app TAG_COMMIT: ${{ github.sha }} run: | + + aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + # Build the Docker image using the specified tags sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . From 59f866a281c564a0081ee8e9eb8c6fb04549b988 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:22:23 +0100 Subject: [PATCH 046/148] testing pipeline --- .github/workflows/runner.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 4cf9b63..a11b921 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -45,11 +45,11 @@ jobs: aws-region: us-east-2 - # - name: Login to Amazon ECR - # id: login-ecr - # run: | - # # Get the ECR login password and use it to log in to the ECR registry - # result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + - name: Login to Amazon ECR + id: login-ecr + run: | + # Get the ECR login password and use it to log in to the ECR registry + sudo result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: @@ -58,7 +58,7 @@ jobs: TAG_COMMIT: ${{ github.sha }} run: | - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + # aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com # Build the Docker image using the specified tags sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . From 0c5e06abc54bec8a377714dcc8bcbc279f44ca5b Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:23:17 +0100 Subject: [PATCH 047/148] testing pipeline --- .github/workflows/runner.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index a11b921..5441831 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -45,11 +45,11 @@ jobs: aws-region: us-east-2 - - name: Login to Amazon ECR - id: login-ecr - run: | - # Get the ECR login password and use it to log in to the ECR registry - sudo result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + # - name: Login to Amazon ECR + # id: login-ecr + # run: | + # # Get the ECR login password and use it to log in to the ECR registry + # sudo result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: @@ -58,7 +58,7 @@ jobs: TAG_COMMIT: ${{ github.sha }} run: | - # aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + sudo aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com # Build the Docker image using the specified tags sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . From 554b1564e7de0fa3866d3b1d8e1fcba06f85f295 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:27:05 +0100 Subject: [PATCH 048/148] testing pipeline --- .github/workflows/runner.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5441831..d7a3d08 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -45,11 +45,11 @@ jobs: aws-region: us-east-2 - # - name: Login to Amazon ECR - # id: login-ecr - # run: | - # # Get the ECR login password and use it to log in to the ECR registry - # sudo result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + - name: Login to Amazon ECR + id: login-ecr + run: | + # Get the ECR login password and use it to log in to the ECR registry + sudo result=$(aws ecr get-login-password --region $AWS_REGION | docker login --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: @@ -58,8 +58,6 @@ jobs: TAG_COMMIT: ${{ github.sha }} run: | - sudo aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - # Build the Docker image using the specified tags sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . @@ -67,7 +65,7 @@ jobs: sudo docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT # Remove the Docker config file after pushing the image - rm -fr ~/.config.json + sudo rm -fr ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID}/config.json $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT >> $GITHUB_OUTPUT From 96ac8dd7ba9e970eb82e7f4e49e8b3cb3b08b28f Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:27:48 +0100 Subject: [PATCH 049/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index d7a3d08..8bb8b6f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -49,7 +49,7 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - sudo result=$(aws ecr get-login-password --region $AWS_REGION | docker login --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + result=$(aws ecr get-login-password --region $AWS_REGION | docker login --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: From 7b03662a6f45e711f7eae72eeeac8c788e68012b Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:28:53 +0100 Subject: [PATCH 050/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 8bb8b6f..8f44ac2 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -49,7 +49,7 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker login --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR env: @@ -62,7 +62,7 @@ jobs: sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . # Push the Docker image to the ECR repository using Docker config from the workspace - sudo docker --config ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID} push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT # Remove the Docker config file after pushing the image sudo rm -fr ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID}/config.json From 6d706131bc98d8df979e9f5d5b5b625f3bd4d063 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:29:04 +0100 Subject: [PATCH 051/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 8f44ac2..28b482f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -65,7 +65,7 @@ jobs: sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT # Remove the Docker config file after pushing the image - sudo rm -fr ${GITHUB_WORKSPACE}/${GITHUB_RUN_ID}/config.json + sudo rm -fr config.json $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT >> $GITHUB_OUTPUT From 6e016366211b3494050008c20d758bc79cad027d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:32:01 +0100 Subject: [PATCH 052/148] testing pipeline --- .github/workflows/runner.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 28b482f..efcd72f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -64,9 +64,6 @@ jobs: # Push the Docker image to the ECR repository using Docker config from the workspace sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - # Remove the Docker config file after pushing the image - sudo rm -fr config.json - $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT >> $GITHUB_OUTPUT From 7a4c3010b85340482662d57c7955dad515a2bd5d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:33:18 +0100 Subject: [PATCH 053/148] testing pipeline --- .github/workflows/runner.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index efcd72f..5571c63 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -64,7 +64,6 @@ jobs: # Push the Docker image to the ECR repository using Docker config from the workspace sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT >> $GITHUB_OUTPUT #Insert vulnerability scan after build From ab3a69b724acb926d74129971c567bbc2f2e5403 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:34:53 +0100 Subject: [PATCH 054/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5571c63..ea34017 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -63,7 +63,7 @@ jobs: # Push the Docker image to the ECR repository using Docker config from the workspace sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT #Insert vulnerability scan after build @@ -89,7 +89,7 @@ jobs: with: task-definition: task-definition.json container-name: my-container - image: ${{ steps.build-image.outputs.image }} + image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 From 9d6b14e1446d75fa11fde2d9733310c0b9b932ff Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:35:34 +0100 Subject: [PATCH 055/148] testing pipeline --- .github/workflows/runner.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index ea34017..a47ee85 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -63,6 +63,7 @@ jobs: # Push the Docker image to the ECR repository using Docker config from the workspace sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT @@ -89,7 +90,7 @@ jobs: with: task-definition: task-definition.json container-name: my-container - image: ${{ steps.build-image.outputs.image }} + image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 From d298d6630a16801df3b58ded5669843a9a31b480 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:36:23 +0100 Subject: [PATCH 056/148] testing pipeline --- .github/workflows/runner.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index a47ee85..de16fc7 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -52,6 +52,7 @@ jobs: result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - name: Build, tag, and push image to Amazon ECR + id: build-image env: ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} ECR_REPOSITORY: go-app From 28878bbfc1bbad97cce3b5ef85101421182f1b27 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:41:48 +0100 Subject: [PATCH 057/148] testing pipeline --- .github/workflows/runner.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index de16fc7..0ea039a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -72,7 +72,7 @@ jobs: - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.28.0 with: - image-ref: ${{ steps.login-ecr.outputs.registry }} + image-ref: ${{ steps.build-image.outputs.image }} format: 'table' exit-code: '1' ignore-unfixed: true @@ -83,7 +83,7 @@ jobs: # #Insert Sign Image verification step before deployment - name: Download task definition run: | - aws ecs describe-task-definition --task-definition my-task-definition-family --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition id: task-def @@ -97,7 +97,6 @@ jobs: uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: my-service - cluster: my-cluster + service: cicdsvc + cluster: cicdtest wait-for-service-stability: true -#### From 81eb85eebd9f8f4ee977ed21aedf5e10a6fd8029 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 17:56:11 +0100 Subject: [PATCH 058/148] testing pipeline --- .github/workflows/runner.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 0ea039a..e30863a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -65,8 +65,7 @@ jobs: # Push the Docker image to the ECR repository using Docker config from the workspace sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT - + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT #Insert vulnerability scan after build - name: Run Trivy vulnerability scanner @@ -80,6 +79,15 @@ jobs: severity: 'CRITICAL,HIGH' + - name: Push image to Amazon ECR + id: push-image + run: | + + # Push the Docker image to the ECR repository using Docker config from the workspace + sudo docker push ${{ steps.build-image.outputs.image }} + + + # #Insert Sign Image verification step before deployment - name: Download task definition run: | From 0c9ae3e66a2fed37a280ae48f72ec8c3b6734823 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:09:51 +0100 Subject: [PATCH 059/148] testing pipeline --- .github/workflows/runner.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index e30863a..b348cf8 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -34,9 +34,9 @@ jobs: with: go-version: '1.23.x' - name: Install dependencies - run: go get ./cmd/loadgen/ && ls -la - - name: Unit Test - run: go test -v -cover ./... + run: go get ./cmd/loadgen/ + # - name: Unit Test + # run: go test -v -cover ./... - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: From e2dda9d265a711d84123b9821415b00f094b27c2 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:25:28 +0100 Subject: [PATCH 060/148] testing pipeline --- .github/workflows/runner.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index b348cf8..7120ba1 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -60,10 +60,10 @@ jobs: run: | # Build the Docker image using the specified tags - sudo docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . # Push the Docker image to the ECR repository using Docker config from the workspace - sudo docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT @@ -84,7 +84,7 @@ jobs: run: | # Push the Docker image to the ECR repository using Docker config from the workspace - sudo docker push ${{ steps.build-image.outputs.image }} + docker push ${{ steps.build-image.outputs.image }} From ee93cbf72e2bfffba2bd5a18f2c2986fb2609188 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:27:03 +0100 Subject: [PATCH 061/148] testing pipeline --- .github/workflows/runner.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 7120ba1..1dccad0 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -61,10 +61,6 @@ jobs: # Build the Docker image using the specified tags docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - - # Push the Docker image to the ECR repository using Docker config from the workspace - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT #Insert vulnerability scan after build From cb6ddcf0fff09755ae2d74143f9166b70c09e3ca Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:31:20 +0100 Subject: [PATCH 062/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 1dccad0..15aeeae 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -73,6 +73,10 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' - name: Push image to Amazon ECR From 72bf48c7f0c65ae9ab286ffbbc2c20ae598355fe Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:33:59 +0100 Subject: [PATCH 063/148] testing pipeline --- .github/workflows/runner.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 15aeeae..133c99a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -63,7 +63,6 @@ jobs: docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - #Insert vulnerability scan after build - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.28.0 with: From c033dcf4781d1d4cabf81d7c7a393e4927c5b4a0 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:37:13 +0100 Subject: [PATCH 064/148] testing pipeline --- .github/workflows/runner.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 133c99a..c86f558 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -23,7 +23,7 @@ jobs: runs-on: self-hosted steps: - # Checkout and test app code +# Checkout and test app code - name: Check out code uses: actions/checkout@v2 with: @@ -44,14 +44,15 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - +# Registry authentication - name: Login to Amazon ECR id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - - - name: Build, tag, and push image to Amazon ECR + +# Build and tag + - name: Build and tag image to Amazon ECR id: build-image env: ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} @@ -63,6 +64,7 @@ jobs: docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT +# Security Testing - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.28.0 with: @@ -87,7 +89,7 @@ jobs: - # #Insert Sign Image verification step before deployment +#Insert Sign Image verification step before deployment - name: Download task definition run: | aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json From fbdd069738b38c017a9540760536aa51ff060a63 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:37:45 +0100 Subject: [PATCH 065/148] testing pipeline --- .github/workflows/runner.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index c86f558..d91d4bc 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -23,7 +23,7 @@ jobs: runs-on: self-hosted steps: -# Checkout and test app code + # Checkout and test app code - name: Check out code uses: actions/checkout@v2 with: @@ -44,14 +44,14 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 -# Registry authentication + # Registry authentication - name: Login to Amazon ECR id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) -# Build and tag + # Build and tag - name: Build and tag image to Amazon ECR id: build-image env: @@ -64,7 +64,7 @@ jobs: docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT -# Security Testing + # Security Testing - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.28.0 with: @@ -89,7 +89,7 @@ jobs: -#Insert Sign Image verification step before deployment + #Insert Sign Image verification step before deployment - name: Download task definition run: | aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json From 249afc154c83b3dc83bcbc5f0321f1f60e8f1eae Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:38:43 +0100 Subject: [PATCH 066/148] testing pipeline --- .github/workflows/runner.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index d91d4bc..9b17963 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -64,7 +64,6 @@ jobs: docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - # Security Testing - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.28.0 with: @@ -79,7 +78,6 @@ jobs: with: sarif_file: 'trivy-results.sarif' - - name: Push image to Amazon ECR id: push-image run: | @@ -88,7 +86,6 @@ jobs: docker push ${{ steps.build-image.outputs.image }} - #Insert Sign Image verification step before deployment - name: Download task definition run: | From 08ecc17301321b29add2907fa456617033615acb Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:39:22 +0100 Subject: [PATCH 067/148] testing pipeline --- .github/workflows/runner.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 9b17963..a246448 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -73,10 +73,10 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: 'trivy-results.sarif' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' - name: Push image to Amazon ECR id: push-image From 01ae244bb960d835c51df46ea18be3d46c11307d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:48:56 +0100 Subject: [PATCH 068/148] testing pipeline --- .github/workflows/runner.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index a246448..0e621bd 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -64,15 +64,15 @@ jobs: docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + # Scan Artefacts - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.28.0 with: image-ref: ${{ steps.build-image.outputs.image }} format: 'table' - exit-code: '1' + # exit-code: '1' #Error in case of vulnerabilities ignore-unfixed: true - vuln-type: 'os,library' - severity: 'CRITICAL,HIGH' + severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v3 with: From 624742aa10b4a20033c4183787eb040d6b0bd39e Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 18:51:48 +0100 Subject: [PATCH 069/148] testing pipeline --- .github/workflows/runner.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 0e621bd..c0367a1 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -73,10 +73,6 @@ jobs: # exit-code: '1' #Error in case of vulnerabilities ignore-unfixed: true severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: 'trivy-results.sarif' - name: Push image to Amazon ECR id: push-image From 486066592038425c02b787cca9e187192bc4da9c Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:05:23 +0100 Subject: [PATCH 070/148] testing pipeline --- .github/workflows/runner.yaml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index c0367a1..936e707 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,16 +1,3 @@ -# name: do-the-job -# on: push -# jobs: -# do-the-job: -# name: Do the job on the runner -# runs-on: self-hosted -# steps: -# - uses: actions/checkout@v2 -# - name: Hello World -# run: | -# echo 'Hello World!' -# sleep 15 -# echo 'Goodbye!' name: Deploy to ECR on: push @@ -79,8 +66,7 @@ jobs: run: | # Push the Docker image to the ECR repository using Docker config from the workspace - docker push ${{ steps.build-image.outputs.image }} - + sudo docker push ${{ steps.build-image.outputs.image }} #Insert Sign Image verification step before deployment - name: Download task definition From 630a9bfadaf3f297646287e9d0f18c18c77c1a84 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:07:52 +0100 Subject: [PATCH 071/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 936e707..65c05f5 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -66,7 +66,7 @@ jobs: run: | # Push the Docker image to the ECR repository using Docker config from the workspace - sudo docker push ${{ steps.build-image.outputs.image }} + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . #Insert Sign Image verification step before deployment - name: Download task definition From 8a96ff46dc161fd1145acbdfb1c0131b23565c09 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:09:22 +0100 Subject: [PATCH 072/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 65c05f5..c37c8fc 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -63,6 +63,10 @@ jobs: - name: Push image to Amazon ECR id: push-image + env: + ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} + ECR_REPOSITORY: go-app + TAG_COMMIT: ${{ github.sha }} run: | # Push the Docker image to the ECR repository using Docker config from the workspace From 5e46b24bb1473dfd0374f0b83e0debdc7790efd8 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:11:16 +0100 Subject: [PATCH 073/148] testing pipeline --- .github/workflows/runner.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index c37c8fc..698087e 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -4,6 +4,10 @@ on: push env: AWS_REGION: eu-west-1 AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} + ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} + TAG_COMMIT: ${{ github.sha }} + ECR_REPOSITORY: go-app + jobs: build: name: CI @@ -41,10 +45,6 @@ jobs: # Build and tag - name: Build and tag image to Amazon ECR id: build-image - env: - ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} - ECR_REPOSITORY: go-app - TAG_COMMIT: ${{ github.sha }} run: | # Build the Docker image using the specified tags @@ -63,10 +63,6 @@ jobs: - name: Push image to Amazon ECR id: push-image - env: - ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} - ECR_REPOSITORY: go-app - TAG_COMMIT: ${{ github.sha }} run: | # Push the Docker image to the ECR repository using Docker config from the workspace From e274da828737cdb0e224fea4ec7d1edb7a9c269c Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:11:38 +0100 Subject: [PATCH 074/148] testing pipeline --- .github/workflows/runner.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 698087e..75c423e 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -72,6 +72,7 @@ jobs: - name: Download task definition run: | aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json + cat task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition id: task-def From bd7ceda9156b2e238cc32316409d4cacbdd79040 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:13:00 +0100 Subject: [PATCH 075/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 75c423e..d5ffd2a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -66,7 +66,7 @@ jobs: run: | # Push the Docker image to the ECR repository using Docker config from the workspace - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT #Insert Sign Image verification step before deployment - name: Download task definition From 4e2f3d5ead9f6d7437b7632f8d2ded8bf9b296c0 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:14:35 +0100 Subject: [PATCH 076/148] testing pipeline --- .github/workflows/runner.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index d5ffd2a..52aab23 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -5,8 +5,8 @@ env: AWS_REGION: eu-west-1 AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} - TAG_COMMIT: ${{ github.sha }} ECR_REPOSITORY: go-app + TAG_COMMIT: ${{ github.sha }} jobs: build: @@ -65,7 +65,9 @@ jobs: id: push-image run: | - # Push the Docker image to the ECR repository using Docker config from the workspace + # Push the Docker image to the ECR repository + echo "$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" + echo "${{ steps.build-image.outputs.image }}" docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT #Insert Sign Image verification step before deployment From 29356a7da01daf0b26d583595e0cbd1fa4c14adf Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:17:27 +0100 Subject: [PATCH 077/148] testing pipeline --- .github/workflows/runner.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 52aab23..9aaa907 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -36,14 +36,9 @@ jobs: aws-region: us-east-2 # Registry authentication - - name: Login to Amazon ECR - id: login-ecr - run: | - # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) # Build and tag - - name: Build and tag image to Amazon ECR + - name: Build and tag image id: build-image run: | @@ -61,6 +56,12 @@ jobs: ignore-unfixed: true severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + - name: Login to Amazon ECR + id: login-ecr + run: | + # Get the ECR login password and use it to log in to the ECR registry + result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + - name: Push image to Amazon ECR id: push-image run: | From 486f8d1caefbbe658c4caa145ccc8220b05bdba4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:19:47 +0100 Subject: [PATCH 078/148] testing pipeline --- .github/workflows/runner.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 9aaa907..680f5ef 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -4,9 +4,6 @@ on: push env: AWS_REGION: eu-west-1 AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} - ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} - ECR_REPOSITORY: go-app - TAG_COMMIT: ${{ github.sha }} jobs: build: @@ -64,6 +61,10 @@ jobs: - name: Push image to Amazon ECR id: push-image + env: + ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} + ECR_REPOSITORY: go-app + TAG_COMMIT: ${{ github.sha }} run: | # Push the Docker image to the ECR repository From 39457ae9981a600b948389b9d7fc6a5c3e5a891b Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:20:28 +0100 Subject: [PATCH 079/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 680f5ef..f83b54e 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -37,6 +37,10 @@ jobs: # Build and tag - name: Build and tag image id: build-image + env: + ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} + ECR_REPOSITORY: go-app + TAG_COMMIT: ${{ github.sha }} run: | # Build the Docker image using the specified tags From 5bb87019a12d3c6fbc3b43d21029bb9e14a2d485 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:23:42 +0100 Subject: [PATCH 080/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index f83b54e..2d1c5ee 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -38,7 +38,7 @@ jobs: - name: Build and tag image id: build-image env: - ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} + ECR_REGISTRY: ${{ env.ECR_REGISTRY }} ECR_REPOSITORY: go-app TAG_COMMIT: ${{ github.sha }} run: | @@ -66,7 +66,7 @@ jobs: - name: Push image to Amazon ECR id: push-image env: - ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }} + ECR_REGISTRY: ${{ env.ECR_REGISTRY }} ECR_REPOSITORY: go-app TAG_COMMIT: ${{ github.sha }} run: | From faa1c8dcf8045ae5b1302cdc70ea95a21e30bdf4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:24:33 +0100 Subject: [PATCH 081/148] testing pipeline --- .github/workflows/runner.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 2d1c5ee..8de7881 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -4,6 +4,8 @@ on: push env: AWS_REGION: eu-west-1 AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} + ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com/go-app + ECR_REPOSITORY: go-app jobs: build: @@ -38,8 +40,6 @@ jobs: - name: Build and tag image id: build-image env: - ECR_REGISTRY: ${{ env.ECR_REGISTRY }} - ECR_REPOSITORY: go-app TAG_COMMIT: ${{ github.sha }} run: | @@ -66,8 +66,6 @@ jobs: - name: Push image to Amazon ECR id: push-image env: - ECR_REGISTRY: ${{ env.ECR_REGISTRY }} - ECR_REPOSITORY: go-app TAG_COMMIT: ${{ github.sha }} run: | From b46187e9abdf1ac3b7f92ec4c708e554d6f6968c Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:25:47 +0100 Subject: [PATCH 082/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 8de7881..5bb0709 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -4,7 +4,7 @@ on: push env: AWS_REGION: eu-west-1 AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} - ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com/go-app + ECR_REGISTRY: ${{ secrets.AWS_ACCOUNT }}.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app jobs: From c2d3e202e7669ca0eb371a949d3d74716c5475d4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:28:01 +0100 Subject: [PATCH 083/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5bb0709..5461a81 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -4,7 +4,7 @@ on: push env: AWS_REGION: eu-west-1 AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} - ECR_REGISTRY: ${{ secrets.AWS_ACCOUNT }}.dkr.ecr.eu-west-1.amazonaws.com + ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app jobs: From 915141cd7cbe5dea7fd049596dc96fd0b61ab722 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:29:02 +0100 Subject: [PATCH 084/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5461a81..0d36677 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -3,7 +3,7 @@ name: Deploy to ECR on: push env: AWS_REGION: eu-west-1 - AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }} + AWS_ACCOUNT: 061051224299 ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app From 40c5a8c97fd7d4e8f9c89a6d185c3cc65667d56b Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:33:34 +0100 Subject: [PATCH 085/148] testing pipeline --- .github/workflows/runner.yaml | 152 +++++++++++++++++----------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 0d36677..93ea9dc 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -7,91 +7,91 @@ env: ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app -jobs: - build: - name: CI - runs-on: self-hosted - steps: + jobs: + build: + name: CI + runs-on: self-hosted + steps: - # Checkout and test app code - - name: Check out code - uses: actions/checkout@v2 - with: - repository: 'dilsilva/go-load-generator' + # Checkout and test app code + - name: Check out code + uses: actions/checkout@v2 + with: + repository: 'dilsilva/go-load-generator' - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23.x' - - name: Install dependencies - run: go get ./cmd/loadgen/ - # - name: Unit Test - # run: go test -v -cover ./... - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23.x' + - name: Install dependencies + run: go get ./cmd/loadgen/ + # - name: Unit Test + # run: go test -v -cover ./... + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 - # Registry authentication + # Registry authentication - # Build and tag - - name: Build and tag image - id: build-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | + # Build and tag + - name: Build and tag image + id: build-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | - # Build the Docker image using the specified tags - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + # Build the Docker image using the specified tags + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - # Scan Artefacts - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.28.0 - with: - image-ref: ${{ steps.build-image.outputs.image }} - format: 'table' - # exit-code: '1' #Error in case of vulnerabilities - ignore-unfixed: true - severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + # Scan Artefacts + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ${{ steps.build-image.outputs.image }} + format: 'table' + # exit-code: '1' #Error in case of vulnerabilities + ignore-unfixed: true + severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - - name: Login to Amazon ECR - id: login-ecr - run: | - # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + - name: Login to Amazon ECR + id: login-ecr + run: | + # Get the ECR login password and use it to log in to the ECR registry + result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - - name: Push image to Amazon ECR - id: push-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | + - name: Push image to Amazon ECR + id: push-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | - # Push the Docker image to the ECR repository - echo "$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" - echo "${{ steps.build-image.outputs.image }}" - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + # Push the Docker image to the ECR repository + echo "$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" + echo "${{ steps.build-image.outputs.image }}" + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - #Insert Sign Image verification step before deployment - - name: Download task definition - run: | - aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json - cat task-definition.json + #Insert Sign Image verification step before deployment + - name: Download task definition + run: | + aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json + cat task-definition.json - - name: Fill in the new image ID in the Amazon ECS task definition - id: task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: task-definition.json - container-name: my-container - image: ${{ steps.build-image.outputs.image }} + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: my-container + image: ${{ steps.build-image.outputs.image }} - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v2 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: cicdsvc - cluster: cicdtest - wait-for-service-stability: true + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: cicdsvc + cluster: cicdtest + wait-for-service-stability: true From 36df2916ddce525f51f51c6bd1642ba9186aae41 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:36:07 +0100 Subject: [PATCH 086/148] testing pipeline --- .github/workflows/runner.yaml | 152 +++++++++++++++++----------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 93ea9dc..1becfd0 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -7,91 +7,91 @@ env: ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app - jobs: - build: - name: CI - runs-on: self-hosted - steps: +jobs: + build: + name: CI + runs-on: self-hosted + steps: - # Checkout and test app code - - name: Check out code - uses: actions/checkout@v2 - with: - repository: 'dilsilva/go-load-generator' + # Checkout and test app code + - name: Check out code + uses: actions/checkout@v2 + with: + repository: 'dilsilva/go-load-generator' - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23.x' - - name: Install dependencies - run: go get ./cmd/loadgen/ - # - name: Unit Test - # run: go test -v -cover ./... - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23.x' + - name: Install dependencies + run: go get ./cmd/loadgen/ + # - name: Unit Test + # run: go test -v -cover ./... + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-1 - # Registry authentication + # Registry authentication - # Build and tag - - name: Build and tag image - id: build-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | + # Build and tag + - name: Build and tag image + id: build-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | - # Build the Docker image using the specified tags - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + # Build the Docker image using the specified tags + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - # Scan Artefacts - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.28.0 - with: - image-ref: ${{ steps.build-image.outputs.image }} - format: 'table' - # exit-code: '1' #Error in case of vulnerabilities - ignore-unfixed: true - severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + # Scan Artefacts + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ${{ steps.build-image.outputs.image }} + format: 'table' + # exit-code: '1' #Error in case of vulnerabilities + ignore-unfixed: true + severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - - name: Login to Amazon ECR - id: login-ecr - run: | - # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + - name: Login to Amazon ECR + id: login-ecr + run: | + # Get the ECR login password and use it to log in to the ECR registry + result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - - name: Push image to Amazon ECR - id: push-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | + - name: Push image to Amazon ECR + id: push-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | - # Push the Docker image to the ECR repository - echo "$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" - echo "${{ steps.build-image.outputs.image }}" - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + # Push the Docker image to the ECR repository + echo "$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" + echo "${{ steps.build-image.outputs.image }}" + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - #Insert Sign Image verification step before deployment - - name: Download task definition - run: | - aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json - cat task-definition.json + #Insert Sign Image verification step before deployment + - name: Download task definition + run: | + aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json + cat task-definition.json - - name: Fill in the new image ID in the Amazon ECS task definition - id: task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: task-definition.json - container-name: my-container - image: ${{ steps.build-image.outputs.image }} + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: my-container + image: ${{ steps.build-image.outputs.image }} - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v2 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: cicdsvc - cluster: cicdtest - wait-for-service-stability: true + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: cicdsvc + cluster: cicdtest + wait-for-service-stability: true From 1188aab35c14bb0118ea1aa5287ec941c84788c8 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:37:04 +0100 Subject: [PATCH 087/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 1becfd0..06acb9c 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -32,7 +32,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 + aws-region: $AWS_ACCOUNT # Registry authentication From a25848abce4782f75543cd0cf3d82a97117c79c9 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:38:34 +0100 Subject: [PATCH 088/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 06acb9c..c772732 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -32,7 +32,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: $AWS_ACCOUNT + aws-region: ${{ env.AWS_REGION}} # Registry authentication From ba404f185754c0d1dd3facdf1ecf435d0e1f921c Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:39:58 +0100 Subject: [PATCH 089/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index c772732..b8acfba 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -32,7 +32,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION}} + aws-region: ${{ env.AWS_REGION }} # Registry authentication From 0cffcecce5e230b5f4bf73730fcdf544d7759cbf Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:42:53 +0100 Subject: [PATCH 090/148] testing pipeline --- .github/workflows/runner.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index b8acfba..9312d4b 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -27,14 +27,6 @@ jobs: run: go get ./cmd/loadgen/ # - name: Unit Test # run: go test -v -cover ./... - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - # Registry authentication # Build and tag - name: Build and tag image @@ -53,10 +45,17 @@ jobs: with: image-ref: ${{ steps.build-image.outputs.image }} format: 'table' + hide-progress: true # exit-code: '1' #Error in case of vulnerabilities ignore-unfixed: true severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} - name: Login to Amazon ECR id: login-ecr run: | From b5b76b1df48398d016baf7f08790d5825cfb0cc6 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:46:58 +0100 Subject: [PATCH 091/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 9312d4b..c666bb4 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -61,7 +61,7 @@ jobs: run: | # Get the ECR login password and use it to log in to the ECR registry result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - + echo $result - name: Push image to Amazon ECR id: push-image env: From 579d306a1b5d3bd83f0db6a5abf9eb395c52b519 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:48:54 +0100 Subject: [PATCH 092/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index c666bb4..53cfcb9 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -60,8 +60,8 @@ jobs: id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - echo $result + #result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) + aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - name: Push image to Amazon ECR id: push-image env: From 534faefe6a929195d8e6199e95468a92122d1e57 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 19:54:33 +0100 Subject: [PATCH 093/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 53cfcb9..9b2632a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -61,7 +61,7 @@ jobs: run: | # Get the ECR login password and use it to log in to the ECR registry #result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + echo $(aws ecr get-login-password --region $AWS_REGION)|docker login --password-stdin --username AWS $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - name: Push image to Amazon ECR id: push-image env: From dfc99a5ba1c0cb2b57e4ad0f6a64673c2f793a1e Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 20:02:28 +0100 Subject: [PATCH 094/148] testing pipeline --- .github/workflows/runner.yaml | 62 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 9b2632a..cb52762 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -14,41 +14,41 @@ jobs: steps: # Checkout and test app code - - name: Check out code - uses: actions/checkout@v2 - with: - repository: 'dilsilva/go-load-generator' + # - name: Check out code + # uses: actions/checkout@v2 + # with: + # repository: 'dilsilva/go-load-generator' - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23.x' - - name: Install dependencies - run: go get ./cmd/loadgen/ - # - name: Unit Test - # run: go test -v -cover ./... + # - name: Setup Go + # uses: actions/setup-go@v5 + # with: + # go-version: '1.23.x' + # - name: Install dependencies + # run: go get ./cmd/loadgen/ + # # - name: Unit Test + # # run: go test -v -cover ./... - # Build and tag - - name: Build and tag image - id: build-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | + # # Build and tag + # - name: Build and tag image + # id: build-image + # env: + # TAG_COMMIT: ${{ github.sha }} + # run: | - # Build the Docker image using the specified tags - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + # # Build the Docker image using the specified tags + # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + # echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - # Scan Artefacts - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.28.0 - with: - image-ref: ${{ steps.build-image.outputs.image }} - format: 'table' - hide-progress: true - # exit-code: '1' #Error in case of vulnerabilities - ignore-unfixed: true - severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + # # Scan Artefacts + # - name: Run Trivy vulnerability scanner + # uses: aquasecurity/trivy-action@0.28.0 + # with: + # image-ref: ${{ steps.build-image.outputs.image }} + # format: 'table' + # hide-progress: true + # # exit-code: '1' #Error in case of vulnerabilities + # ignore-unfixed: true + # severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 From 13138baf0f5e8a70de244ef28f2b5378598caf56 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 20:51:50 +0100 Subject: [PATCH 095/148] testing pipeline --- .github/workflows/runner.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index cb52762..14770c2 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -56,6 +56,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} + - name: Login to Amazon ECR id: login-ecr run: | @@ -69,8 +70,6 @@ jobs: run: | # Push the Docker image to the ECR repository - echo "$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" - echo "${{ steps.build-image.outputs.image }}" docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT #Insert Sign Image verification step before deployment From 3a21592c7c92d62cd866d3772b70f8e7a20e628c Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 20:55:07 +0100 Subject: [PATCH 096/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 14770c2..5845848 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -48,7 +48,7 @@ jobs: # hide-progress: true # # exit-code: '1' #Error in case of vulnerabilities # ignore-unfixed: true - # severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + # severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -56,7 +56,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} - + - name: Login to Amazon ECR id: login-ecr run: | From cb8a4e39e59f0eec9bedc1844b59f665e50a63fa Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:00:51 +0100 Subject: [PATCH 097/148] testing pipeline --- .github/workflows/runner.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5845848..22bdf42 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -50,20 +50,19 @@ jobs: # ignore-unfixed: true # severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ env.AWS_REGION }} - - name: Login to Amazon ECR + - name: Login to ECR id: login-ecr run: | # Get the ECR login password and use it to log in to the ECR registry - #result=$(aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com) - echo $(aws ecr get-login-password --region $AWS_REGION)|docker login --password-stdin --username AWS $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - - name: Push image to Amazon ECR + # aws ecr get-login-password --region $AWS_REGION|docker login --password-stdin --username AWS $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + aws ecr get-login-password --region eu-west-1 | docker login --password-stdin --username AWS 061051224299.dkr.ecr.eu-west-1.amazonaws.com - name: Push image to Amazon ECR id: push-image env: TAG_COMMIT: ${{ github.sha }} From 8f4331356875be375cae2cb410c8bcada08647c1 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:02:36 +0100 Subject: [PATCH 098/148] testing pipeline --- .github/workflows/runner.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 22bdf42..0041f0b 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -63,6 +63,8 @@ jobs: # Get the ECR login password and use it to log in to the ECR registry # aws ecr get-login-password --region $AWS_REGION|docker login --password-stdin --username AWS $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com aws ecr get-login-password --region eu-west-1 | docker login --password-stdin --username AWS 061051224299.dkr.ecr.eu-west-1.amazonaws.com - name: Push image to Amazon ECR + + - name: Push image id: push-image env: TAG_COMMIT: ${{ github.sha }} From 4ea277b8a8f8473f4b6c6cd94c6a5b90aa229f70 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:03:56 +0100 Subject: [PATCH 099/148] testing pipeline --- .github/workflows/runner.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 0041f0b..fb5669c 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -73,7 +73,6 @@ jobs: # Push the Docker image to the ECR repository docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - #Insert Sign Image verification step before deployment - name: Download task definition run: | aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json From 5e1a1db4c19ac0897fd586cab59dd28ee3d2c3fc Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:04:32 +0100 Subject: [PATCH 100/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index fb5669c..07c3a7f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -62,7 +62,7 @@ jobs: run: | # Get the ECR login password and use it to log in to the ECR registry # aws ecr get-login-password --region $AWS_REGION|docker login --password-stdin --username AWS $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - aws ecr get-login-password --region eu-west-1 | docker login --password-stdin --username AWS 061051224299.dkr.ecr.eu-west-1.amazonaws.com - name: Push image to Amazon ECR + aws ecr get-login-password --region eu-west-1 | docker login --password-stdin --username AWS 061051224299.dkr.ecr.eu-west-1.amazonaws.com - name: Push image id: push-image From b6984d47c15ef404bc1c785f34b0f5269db3348d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:06:07 +0100 Subject: [PATCH 101/148] testing pipeline --- .github/workflows/runner.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 07c3a7f..ab77314 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -62,8 +62,11 @@ jobs: run: | # Get the ECR login password and use it to log in to the ECR registry # aws ecr get-login-password --region $AWS_REGION|docker login --password-stdin --username AWS $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - aws ecr get-login-password --region eu-west-1 | docker login --password-stdin --username AWS 061051224299.dkr.ecr.eu-west-1.amazonaws.com - + # result(aws ecr get-login-password --region eu-west-1 | docker login --password-stdin --username AWS 061051224299.dkr.ecr.eu-west-1.amazonaws.com + aws ecr get-login-password \ + --region $AWS_REGION | docker login \ + --username AWS \ + --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - name: Push image id: push-image env: From 86238119636ee0fa9d158d968fc9beca5954664a Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:07:15 +0100 Subject: [PATCH 102/148] testing pipeline --- .github/workflows/runner.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index ab77314..238282e 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -2,8 +2,8 @@ name: Deploy to ECR on: push env: - AWS_REGION: eu-west-1 AWS_ACCOUNT: 061051224299 + AWS_REGION: eu-west-1 ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app @@ -59,10 +59,11 @@ jobs: - name: Login to ECR id: login-ecr + env: + AWS_ACCOUNT: 061051224299 + AWS_REGION: eu-west-1 run: | # Get the ECR login password and use it to log in to the ECR registry - # aws ecr get-login-password --region $AWS_REGION|docker login --password-stdin --username AWS $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - # result(aws ecr get-login-password --region eu-west-1 | docker login --password-stdin --username AWS 061051224299.dkr.ecr.eu-west-1.amazonaws.com aws ecr get-login-password \ --region $AWS_REGION | docker login \ --username AWS \ From 0b372c188191f99615837b9f67a7f992e87c01e6 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:07:56 +0100 Subject: [PATCH 103/148] testing pipeline --- .github/workflows/runner.yaml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 238282e..95f6f9a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,11 +1,6 @@ name: Deploy to ECR on: push -env: - AWS_ACCOUNT: 061051224299 - AWS_REGION: eu-west-1 - ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com - ECR_REPOSITORY: go-app jobs: build: @@ -63,11 +58,11 @@ jobs: AWS_ACCOUNT: 061051224299 AWS_REGION: eu-west-1 run: | - # Get the ECR login password and use it to log in to the ECR registry - aws ecr get-login-password \ - --region $AWS_REGION | docker login \ - --username AWS \ - --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + # Get the ECR login password and use it to log in to the ECR registry + aws ecr get-login-password \ + --region $AWS_REGION | docker login \ + --username AWS \ + --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com - name: Push image id: push-image env: From 90d817cedc10dac00e70dba0f2a5f4147f1e86c0 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:10:31 +0100 Subject: [PATCH 104/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 95f6f9a..08bf08d 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -62,7 +62,7 @@ jobs: aws ecr get-login-password \ --region $AWS_REGION | docker login \ --username AWS \ - --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com + --password-stdin $ECR_REGISTRY - name: Push image id: push-image env: From 70feb8596fa69155c6a137fa6c18ede9dc5e3ed3 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:11:27 +0100 Subject: [PATCH 105/148] testing pipeline --- .github/workflows/runner.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 08bf08d..e157845 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,6 +1,11 @@ name: Deploy to ECR on: push +env: + AWS_ACCOUNT: 061051224299 + AWS_REGION: eu-west-1 + ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com + ECR_REPOSITORY: go-app jobs: build: @@ -60,9 +65,9 @@ jobs: run: | # Get the ECR login password and use it to log in to the ECR registry aws ecr get-login-password \ - --region $AWS_REGION | docker login \ + --region 061051224299 | docker login \ --username AWS \ - --password-stdin $ECR_REGISTRY + --password-stdin 061051224299.dkr.ecr.eu-west-1.amazonaws.com - name: Push image id: push-image env: From 4e092e29543fa9aaf3a3cdd774ff29dfb3710faf Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:12:10 +0100 Subject: [PATCH 106/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index e157845..d0380d0 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -65,7 +65,7 @@ jobs: run: | # Get the ECR login password and use it to log in to the ECR registry aws ecr get-login-password \ - --region 061051224299 | docker login \ + --region eu-west-1 | docker login \ --username AWS \ --password-stdin 061051224299.dkr.ecr.eu-west-1.amazonaws.com - name: Push image From 2a85cba534e98c6f026f6cd1eda8f162e6907bfe Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:13:16 +0100 Subject: [PATCH 107/148] testing pipeline --- .github/workflows/runner.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index d0380d0..9d1b4b5 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -14,10 +14,10 @@ jobs: steps: # Checkout and test app code - # - name: Check out code - # uses: actions/checkout@v2 - # with: - # repository: 'dilsilva/go-load-generator' + - name: Check out code + uses: actions/checkout@v2 + with: + repository: 'dilsilva/go-load-generator' # - name: Setup Go # uses: actions/setup-go@v5 @@ -28,16 +28,16 @@ jobs: # # - name: Unit Test # # run: go test -v -cover ./... - # # Build and tag - # - name: Build and tag image - # id: build-image - # env: - # TAG_COMMIT: ${{ github.sha }} - # run: | + # Build and tag + - name: Build and tag image + id: build-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | - # # Build the Docker image using the specified tags - # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - # echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + # Build the Docker image using the specified tags + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT # # Scan Artefacts # - name: Run Trivy vulnerability scanner From 140c5dc24252876a77ea257d9cc45052665b665b Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:15:38 +0100 Subject: [PATCH 108/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 9d1b4b5..cfefa25 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -87,7 +87,7 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: my-container + container-name: nginx image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition From 1a663f34cc5aef42831cc328277139fd7cb3d8df Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:24:38 +0100 Subject: [PATCH 109/148] testing pipeline --- .github/workflows/runner.yaml | 68 +++++++++++++++++------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index cfefa25..5ea6988 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -12,51 +12,49 @@ jobs: name: CI runs-on: self-hosted steps: - - # Checkout and test app code - name: Check out code uses: actions/checkout@v2 with: repository: 'dilsilva/go-load-generator' - # - name: Setup Go - # uses: actions/setup-go@v5 - # with: - # go-version: '1.23.x' - # - name: Install dependencies - # run: go get ./cmd/loadgen/ - # # - name: Unit Test - # # run: go test -v -cover ./... +##### +## Build and test Go binary + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23.x' + - name: Install dependencies + run: go get . + - name: Build + run: go build -v ./... + - name: Test with the Go CLI + run: go test - # Build and tag +##### +## Build and tag image - name: Build and tag image id: build-image env: TAG_COMMIT: ${{ github.sha }} run: | - - # Build the Docker image using the specified tags docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - # # Scan Artefacts - # - name: Run Trivy vulnerability scanner - # uses: aquasecurity/trivy-action@0.28.0 - # with: - # image-ref: ${{ steps.build-image.outputs.image }} - # format: 'table' - # hide-progress: true - # # exit-code: '1' #Error in case of vulnerabilities - # ignore-unfixed: true - # severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities +##### +## Scan image for security flaws + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ${{ steps.build-image.outputs.image }} + format: 'table' + hide-progress: true + # exit-code: '1' #Error in case of vulnerabilities + ignore-unfixed: true + severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ env.AWS_REGION }} +##### +## Push the Docker image to the ECR repository - name: Login to ECR id: login-ecr env: @@ -65,18 +63,20 @@ jobs: run: | # Get the ECR login password and use it to log in to the ECR registry aws ecr get-login-password \ - --region eu-west-1 | docker login \ + --region $AWS_REGION | docker login \ --username AWS \ - --password-stdin 061051224299.dkr.ecr.eu-west-1.amazonaws.com + --password-stdin $ECR_REGISTRY + - name: Push image id: push-image env: TAG_COMMIT: ${{ github.sha }} run: | + docker push ${{ steps.build-image.outputs.image }} + # docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - # Push the Docker image to the ECR repository - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - +##### +## Deploy image to ECS - name: Download task definition run: | aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json From c4b0253705929da997b8d591f2c30d919e80b183 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:26:15 +0100 Subject: [PATCH 110/148] testing pipeline --- .github/workflows/runner.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 5ea6988..b4db03f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -88,7 +88,8 @@ jobs: with: task-definition: task-definition.json container-name: nginx - image: ${{ steps.build-image.outputs.image }} + # image: ${{ steps.build-image.outputs.image }} + image: public.ecr.aws/nginx/nginx:alpine-slim - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 From a538c9e70b54deae370907b32d8489abfbd1338a Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:26:43 +0100 Subject: [PATCH 111/148] testing pipeline --- .github/workflows/runner.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index b4db03f..a8a7b32 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -12,10 +12,10 @@ jobs: name: CI runs-on: self-hosted steps: - - name: Check out code - uses: actions/checkout@v2 - with: - repository: 'dilsilva/go-load-generator' + - name: Check out code + uses: actions/checkout@v2 + with: + repository: 'dilsilva/go-load-generator' ##### ## Build and test Go binary From 1a45d8e6e8e0dc1a448641598dd6f9599bded210 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:41:59 +0100 Subject: [PATCH 112/148] testing pipeline --- .github/workflows/runner.yaml | 121 ++++++++++++++++------------------ 1 file changed, 57 insertions(+), 64 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index a8a7b32..59b27c4 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -13,12 +13,11 @@ jobs: runs-on: self-hosted steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: 'dilsilva/go-load-generator' -##### -## Build and test Go binary + - name: Setup Go uses: actions/setup-go@v5 with: @@ -30,71 +29,65 @@ jobs: - name: Test with the Go CLI run: go test -##### -## Build and tag image - - name: Build and tag image - id: build-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT -##### -## Scan image for security flaws - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.28.0 - with: - image-ref: ${{ steps.build-image.outputs.image }} - format: 'table' - hide-progress: true - # exit-code: '1' #Error in case of vulnerabilities - ignore-unfixed: true - severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + - name: Build and tag image + id: build-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ${{ steps.build-image.outputs.image }} + format: 'table' + hide-progress: true + # exit-code: '1' #Error in case of vulnerabilities + ignore-unfixed: true + severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + -##### -## Push the Docker image to the ECR repository - - name: Login to ECR - id: login-ecr - env: - AWS_ACCOUNT: 061051224299 - AWS_REGION: eu-west-1 - run: | - # Get the ECR login password and use it to log in to the ECR registry - aws ecr get-login-password \ - --region $AWS_REGION | docker login \ - --username AWS \ - --password-stdin $ECR_REGISTRY + - name: Login to ECR + id: login-ecr + env: + AWS_ACCOUNT: 061051224299 + AWS_REGION: eu-west-1 + run: | + # Get the ECR login password and use it to log in to the ECR registry + aws ecr get-login-password \ + --region $AWS_REGION | docker login \ + --username AWS \ + --password-stdin $ECR_REGISTRY - - name: Push image - id: push-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | - docker push ${{ steps.build-image.outputs.image }} - # docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + - name: Push image + id: push-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | + docker push ${{ steps.build-image.outputs.image }} + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT -##### -## Deploy image to ECS - - name: Download task definition - run: | - aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json - cat task-definition.json - - name: Fill in the new image ID in the Amazon ECS task definition - id: task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: task-definition.json - container-name: nginx - # image: ${{ steps.build-image.outputs.image }} - image: public.ecr.aws/nginx/nginx:alpine-slim + - name: Download task definition + run: | + aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json + cat task-definition.json - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v2 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: cicdsvc - cluster: cicdtest - wait-for-service-stability: true + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: nginx + image: ${{ steps.build-image.outputs.image }} + + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: cicdsvc + cluster: cicdtest + wait-for-service-stability: true From 7091aada038363af13817f561034bd34c5db5ee9 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:43:47 +0100 Subject: [PATCH 113/148] testing pipeline --- .github/workflows/runner.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 59b27c4..31f398d 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -23,7 +23,7 @@ jobs: with: go-version: '1.23.x' - name: Install dependencies - run: go get . + run: go get ./cmd/loadgen/ - name: Build run: go build -v ./... - name: Test with the Go CLI @@ -48,8 +48,6 @@ jobs: ignore-unfixed: true severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - - - name: Login to ECR id: login-ecr env: @@ -70,7 +68,6 @@ jobs: docker push ${{ steps.build-image.outputs.image }} docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT - - name: Download task definition run: | aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json From 2471ad7aecc2b25a7c7b70d177898e6b18d840bc Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:45:23 +0100 Subject: [PATCH 114/148] testing pipeline --- .github/workflows/runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 31f398d..f809186 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -25,9 +25,9 @@ jobs: - name: Install dependencies run: go get ./cmd/loadgen/ - name: Build - run: go build -v ./... + run: go build -o loadgen -ldflags="-s -w" ./cmd/loadgen/ - name: Test with the Go CLI - run: go test + run: go test -v -cover ./... - name: Build and tag image From 3cfbe74d1fe69942c89021770cb70d66e6fce7e4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:50:29 +0100 Subject: [PATCH 115/148] testing pipeline --- .github/workflows/runner.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index f809186..4e7e393 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -70,7 +70,7 @@ jobs: - name: Download task definition run: | - aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition ecs-sample --query taskDefinition > task-definition.json cat task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition @@ -78,13 +78,13 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: nginx - image: ${{ steps.build-image.outputs.image }} + container-name: ecs-sample + image: public.ecr.aws/aws-containers/ecsdemo-frontend:776fd50 - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: cicdsvc + service: ecs-sample cluster: cicdtest wait-for-service-stability: true From 3343996d14ef3a5ad229b6f2900c2b44d8349e27 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:51:10 +0100 Subject: [PATCH 116/148] testing pipeline --- .github/workflows/runner.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 4e7e393..4f53b2b 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -24,8 +24,6 @@ jobs: go-version: '1.23.x' - name: Install dependencies run: go get ./cmd/loadgen/ - - name: Build - run: go build -o loadgen -ldflags="-s -w" ./cmd/loadgen/ - name: Test with the Go CLI run: go test -v -cover ./... From bb4ea18ddc8822417b07bb9be146cdd12a3ba0b2 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 21:55:11 +0100 Subject: [PATCH 117/148] testing pipeline --- .github/workflows/runner.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 4f53b2b..ba7b828 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -18,14 +18,14 @@ jobs: repository: 'dilsilva/go-load-generator' - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23.x' - - name: Install dependencies - run: go get ./cmd/loadgen/ - - name: Test with the Go CLI - run: go test -v -cover ./... + # - name: Setup Go + # uses: actions/setup-go@v5 + # with: + # go-version: '1.23.x' + # - name: Install dependencies + # run: go get ./cmd/loadgen/ + # - name: Test with the Go CLI + # run: go test -v -cover ./... - name: Build and tag image From 211109525857daa565448226279a7697dfb7d439 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Fri, 6 Dec 2024 22:06:23 +0100 Subject: [PATCH 118/148] testing pipeline --- .github/workflows/runner.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index ba7b828..16cba02 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -68,7 +68,7 @@ jobs: - name: Download task definition run: | - aws ecs describe-task-definition --task-definition ecs-sample --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json cat task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition @@ -76,13 +76,13 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: ecs-sample - image: public.ecr.aws/aws-containers/ecsdemo-frontend:776fd50 + container-name: nginx + image: nginx/nginx - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: ecs-sample + service: cicdsvc cluster: cicdtest wait-for-service-stability: true From 78a46e1497a643061f9a0ada230e7be2cba3a786 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 01:08:50 +0100 Subject: [PATCH 119/148] testing pipeline --- .github/workflows/runner.yaml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 16cba02..0bb1b6a 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -14,25 +14,13 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 - with: - repository: 'dilsilva/go-load-generator' - - - # - name: Setup Go - # uses: actions/setup-go@v5 - # with: - # go-version: '1.23.x' - # - name: Install dependencies - # run: go get ./cmd/loadgen/ - # - name: Test with the Go CLI - # run: go test -v -cover ./... - - name: Build and tag image id: build-image env: TAG_COMMIT: ${{ github.sha }} run: | + cd app/ docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT From 117a711ff99548716f0beb57f1485c1c66d8a133 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 01:11:13 +0100 Subject: [PATCH 120/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 0bb1b6a..e45ceb5 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -21,7 +21,7 @@ jobs: TAG_COMMIT: ${{ github.sha }} run: | cd app/ - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - name: Run Trivy vulnerability scanner From 9e230128efebde59a9437608d69c553cebf3d92d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 01:12:00 +0100 Subject: [PATCH 121/148] testing pipeline --- app/Dockerfile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 app/Dockerfile diff --git a/app/Dockerfile b/app/Dockerfile new file mode 100644 index 0000000..c752f34 --- /dev/null +++ b/app/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.23 +WORKDIR /src + +COPY < hello k8s ")) +} +EOF +RUN ls +RUN go mod init server +RUN go mod tidy +RUN go build -o hello + +CMD ["/src/hello"] \ No newline at end of file From 0d08e502bdd432cf95bda3ac0604d24bcf30668a Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 01:43:23 +0100 Subject: [PATCH 122/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index e45ceb5..af974da 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -43,7 +43,7 @@ jobs: # Get the ECR login password and use it to log in to the ECR registry aws ecr get-login-password \ --region $AWS_REGION | docker login \ - --username AWS \ + --username AWS \ --password-stdin $ECR_REGISTRY - name: Push image From 78d651fc3dcf79b5dff5535c99558fc97feeda1d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:05:09 +0100 Subject: [PATCH 123/148] testing pipeline --- .github/workflows/runner.yaml | 44 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index af974da..08d0d02 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -12,27 +12,27 @@ jobs: name: CI runs-on: self-hosted steps: - - name: Check out code - uses: actions/checkout@v4 + # - name: Check out code + # uses: actions/checkout@v4 - - name: Build and tag image - id: build-image - env: - TAG_COMMIT: ${{ github.sha }} - run: | - cd app/ - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + # - name: Build and tag image + # id: build-image + # env: + # TAG_COMMIT: ${{ github.sha }} + # run: | + # cd app/ + # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + # echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.28.0 - with: - image-ref: ${{ steps.build-image.outputs.image }} - format: 'table' - hide-progress: true - # exit-code: '1' #Error in case of vulnerabilities - ignore-unfixed: true - severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + # # - name: Run Trivy vulnerability scanner + # # uses: aquasecurity/trivy-action@0.28.0 + # # with: + # # image-ref: ${{ steps.build-image.outputs.image }} + # # format: 'table' + # # hide-progress: true + # # # exit-code: '1' #Error in case of vulnerabilities + # # ignore-unfixed: true + # # severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - name: Login to ECR id: login-ecr @@ -40,11 +40,7 @@ jobs: AWS_ACCOUNT: 061051224299 AWS_REGION: eu-west-1 run: | - # Get the ECR login password and use it to log in to the ECR registry - aws ecr get-login-password \ - --region $AWS_REGION | docker login \ - --username AWS \ - --password-stdin $ECR_REGISTRY + docker login -u AWS -p $(aws ecr get-login-password --region $AWS_REGION) $ECR_REGISTRY - name: Push image id: push-image From 4d48c1fa98dcb7655797f29e0c1c041f6b11f458 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:06:45 +0100 Subject: [PATCH 124/148] testing pipeline --- .github/workflows/runner.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 08d0d02..df011fc 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -12,17 +12,17 @@ jobs: name: CI runs-on: self-hosted steps: - # - name: Check out code - # uses: actions/checkout@v4 + - name: Check out code + uses: actions/checkout@v4 - # - name: Build and tag image - # id: build-image - # env: - # TAG_COMMIT: ${{ github.sha }} - # run: | - # cd app/ - # docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - # echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT + - name: Build and tag image + id: build-image + env: + TAG_COMMIT: ${{ github.sha }} + run: | + cd app/ + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT # # - name: Run Trivy vulnerability scanner # # uses: aquasecurity/trivy-action@0.28.0 @@ -49,6 +49,7 @@ jobs: run: | docker push ${{ steps.build-image.outputs.image }} docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + sudo rm /root/.docker/config.json - name: Download task definition run: | From b75b8b7cf0f4cec1249f3306496d9508deacb3e5 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:25:18 +0100 Subject: [PATCH 125/148] testing pipeline --- .github/workflows/runner.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index df011fc..3896666 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -22,6 +22,7 @@ jobs: run: | cd app/ docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . + docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT $ECR_REGISTRY/$ECR_REPOSITORY:latest echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT # # - name: Run Trivy vulnerability scanner @@ -48,7 +49,7 @@ jobs: TAG_COMMIT: ${{ github.sha }} run: | docker push ${{ steps.build-image.outputs.image }} - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT + docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest sudo rm /root/.docker/config.json - name: Download task definition @@ -69,5 +70,5 @@ jobs: with: task-definition: ${{ steps.task-def.outputs.task-definition }} service: cicdsvc - cluster: cicdtest + cluster: surepay-ecs wait-for-service-stability: true From a1c2c83555e04438bc9088a611e0a2d0a21d9612 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:25:29 +0100 Subject: [PATCH 126/148] testing pipeline --- app/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Dockerfile b/app/Dockerfile index c752f34..2eb1910 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -8,7 +8,7 @@ func main() { app := gin.Default() router := app.Group("/") router.GET("/", Hello) - app.Run(":8080") + app.Run(":80") } func Hello(c *gin.Context) { c.Writer.Write([]byte("

hello k8s

")) From e6150bc8d70d914084f13695a0fe3ea87055ced3 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:26:37 +0100 Subject: [PATCH 127/148] testing pipeline --- app/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Dockerfile b/app/Dockerfile index 2eb1910..fdebfe7 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -18,5 +18,6 @@ RUN ls RUN go mod init server RUN go mod tidy RUN go build -o hello +EXPOSE 80 CMD ["/src/hello"] \ No newline at end of file From 2c9ea2f628f6dc713fe680ccf4a3f73323ddcda6 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:28:00 +0100 Subject: [PATCH 128/148] testing pipeline --- .github/workflows/runner.yaml | 8 ++++---- app/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 3896666..e8c511f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -54,7 +54,7 @@ jobs: - name: Download task definition run: | - aws ecs describe-task-definition --task-definition web --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition goserver --query taskDefinition > task-definition.json cat task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition @@ -62,13 +62,13 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: nginx - image: nginx/nginx + container-name: ecs-sample + image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: cicdsvc + service: goserver cluster: surepay-ecs wait-for-service-stability: true diff --git a/app/Dockerfile b/app/Dockerfile index fdebfe7..40de301 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -18,6 +18,6 @@ RUN ls RUN go mod init server RUN go mod tidy RUN go build -o hello -EXPOSE 80 +EXPOSE 80 CMD ["/src/hello"] \ No newline at end of file From 9e345c5fd9a4c6ece4849fabf2f6f6abc67dfe04 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:28:45 +0100 Subject: [PATCH 129/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index e8c511f..eddd125 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -70,5 +70,5 @@ jobs: with: task-definition: ${{ steps.task-def.outputs.task-definition }} service: goserver - cluster: surepay-ecs + cluster: cicdtest-ecs wait-for-service-stability: true From 6249cdb373623662131827e19f0792b239b0afc3 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sat, 7 Dec 2024 02:34:34 +0100 Subject: [PATCH 130/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index eddd125..760bf91 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -71,4 +71,4 @@ jobs: task-definition: ${{ steps.task-def.outputs.task-definition }} service: goserver cluster: cicdtest-ecs - wait-for-service-stability: true + wait-for-service-stability: true \ No newline at end of file From f463c7d5947a5c634b8368871b4c132fc2caa7e7 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 02:46:48 +0100 Subject: [PATCH 131/148] testing pipeline --- .github/workflows/runner.yaml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 760bf91..f1dbc55 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -22,18 +22,18 @@ jobs: run: | cd app/ docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT $ECR_REGISTRY/$ECR_REPOSITORY:latest + docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - # # - name: Run Trivy vulnerability scanner - # # uses: aquasecurity/trivy-action@0.28.0 - # # with: - # # image-ref: ${{ steps.build-image.outputs.image }} - # # format: 'table' - # # hide-progress: true - # # # exit-code: '1' #Error in case of vulnerabilities - # # ignore-unfixed: true - # # severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ${{ steps.build-image.outputs.image }} + format: 'table' + hide-progress: true + # exit-code: '1' #Error in case of vulnerabilities + ignore-unfixed: true + severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities - name: Login to ECR id: login-ecr @@ -49,12 +49,11 @@ jobs: TAG_COMMIT: ${{ github.sha }} run: | docker push ${{ steps.build-image.outputs.image }} - docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest sudo rm /root/.docker/config.json - name: Download task definition run: | - aws ecs describe-task-definition --task-definition goserver --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition surepay-task --query taskDefinition > task-definition.json cat task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition @@ -62,13 +61,13 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: ecs-sample + container-name: goserver image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: goserver - cluster: cicdtest-ecs + service: surepay-service + cluster: surepay-ecs wait-for-service-stability: true \ No newline at end of file From 8e9d08dc758f6cbea4e3442dc426f7281115fc92 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 03:03:30 +0100 Subject: [PATCH 132/148] testing pipeline --- .github/workflows/runner.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index f1dbc55..2a3dd44 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -22,7 +22,6 @@ jobs: run: | cd app/ docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT . - docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$TAG_COMMIT" >> $GITHUB_OUTPUT - name: Run Trivy vulnerability scanner From be569c8feaea1390e17f89d02ee301d5826afa9c Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 03:24:51 +0100 Subject: [PATCH 133/148] testing pipeline --- app/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Dockerfile b/app/Dockerfile index 40de301..af99185 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -11,7 +11,7 @@ func main() { app.Run(":80") } func Hello(c *gin.Context) { - c.Writer.Write([]byte("

hello k8s

")) + c.Writer.Write([]byte("

hello ecs

")) } EOF RUN ls From 9819e74731c3b1e27c090bbdf50f7fe8c4a839f0 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 03:25:00 +0100 Subject: [PATCH 134/148] testing pipeline --- .github/workflows/runner.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 2a3dd44..7df095c 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -69,4 +69,5 @@ jobs: task-definition: ${{ steps.task-def.outputs.task-definition }} service: surepay-service cluster: surepay-ecs + desired-count: 4 wait-for-service-stability: true \ No newline at end of file From 2851e0d3a3089dbfacbae7ba683bc3c4474a48b9 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 04:49:58 +0100 Subject: [PATCH 135/148] testing pipeline --- .github/workflows/runner.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 7df095c..1102cd4 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -33,6 +33,7 @@ jobs: # exit-code: '1' #Error in case of vulnerabilities ignore-unfixed: true severity: 'CRITICAL,HIGH' #Ignore low severity vulnerabilities + cache: 'false' - name: Login to ECR id: login-ecr From a901d16127752b09e816f77f696a9bd1b3676a6a Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 05:58:27 +0100 Subject: [PATCH 136/148] testing pipeline --- .github/workflows/runner.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 1102cd4..f95595e 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -6,6 +6,10 @@ env: AWS_REGION: eu-west-1 ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app + ECS_CONTAINER_NAME: go-app + ECS_SERVICE_NAME: surepay-service + ECS_CLUSTER_NAME: surepay-ecs + jobs: build: @@ -61,14 +65,14 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: goserver + container-name: $ECS_CONTAINER_NAME image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: surepay-service - cluster: surepay-ecs + service: $ECS_SERVICE_NAME + cluster: $ECS_CLUSTER_NAME desired-count: 4 wait-for-service-stability: true \ No newline at end of file From a7c5e2f56c89330a3f45188ead6e84efecea44c9 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 06:05:42 +0100 Subject: [PATCH 137/148] testing pipeline --- .github/workflows/runner.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index f95595e..6187fd7 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -6,16 +6,20 @@ env: AWS_REGION: eu-west-1 ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app - ECS_CONTAINER_NAME: go-app - ECS_SERVICE_NAME: surepay-service - ECS_CLUSTER_NAME: surepay-ecs - jobs: build: name: CI runs-on: self-hosted steps: + - name: Set globals + id: globals + shell: bash + run: | + echo " ECS_CONTAINER_NAME=go-app" >> "${GITHUB_OUTPUT}" + echo " ECS_SERVICE_NAME=surepay-service" >> "${GITHUB_OUTPUT}" + echo " ECS_CLUSTER_NAME=surepay-ecs" >> "${GITHUB_OUTPUT}" + - name: Check out code uses: actions/checkout@v4 @@ -65,14 +69,14 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: $ECS_CONTAINER_NAME + container-name: ${{ steps.globals.outputs.ECS_CONTAINER_NAME }} image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: $ECS_SERVICE_NAME - cluster: $ECS_CLUSTER_NAME + service: ${{ steps.globals.outputs.ECS_SERVICE_NAME }} + cluster: ${{ steps.globals.outputs.ECS_CLUSTER_NAME }} desired-count: 4 wait-for-service-stability: true \ No newline at end of file From aca2ef2b6f14659bb7e59172a6dc13ce7b26dc32 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 06:10:28 +0100 Subject: [PATCH 138/148] testing pipeline --- .github/workflows/runner.yaml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 6187fd7..50fd98c 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -7,19 +7,13 @@ env: ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app + + jobs: build: name: CI runs-on: self-hosted steps: - - name: Set globals - id: globals - shell: bash - run: | - echo " ECS_CONTAINER_NAME=go-app" >> "${GITHUB_OUTPUT}" - echo " ECS_SERVICE_NAME=surepay-service" >> "${GITHUB_OUTPUT}" - echo " ECS_CLUSTER_NAME=surepay-ecs" >> "${GITHUB_OUTPUT}" - - name: Check out code uses: actions/checkout@v4 @@ -69,14 +63,14 @@ jobs: uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: task-definition.json - container-name: ${{ steps.globals.outputs.ECS_CONTAINER_NAME }} + container-name: go-app image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: ${{ steps.globals.outputs.ECS_SERVICE_NAME }} - cluster: ${{ steps.globals.outputs.ECS_CLUSTER_NAME }} + service: surepay-service + cluster: surepay-ecs desired-count: 4 wait-for-service-stability: true \ No newline at end of file From 8ae80b974dc53d616ce3fb4268d519a125ce179d Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 06:11:39 +0100 Subject: [PATCH 139/148] testing pipeline --- .github/workflows/runner.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 50fd98c..38d12f3 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -7,11 +7,9 @@ env: ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com ECR_REPOSITORY: go-app - - jobs: build: - name: CI + name: CICD runs-on: self-hosted steps: - name: Check out code @@ -55,7 +53,7 @@ jobs: - name: Download task definition run: | - aws ecs describe-task-definition --task-definition surepay-task --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition go-app --query taskDefinition > task-definition.json cat task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition From aaa7b35362fccaaf68448d0e67b889019eab53b1 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 06:16:56 +0100 Subject: [PATCH 140/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 38d12f3..52b848f 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -53,7 +53,7 @@ jobs: - name: Download task definition run: | - aws ecs describe-task-definition --task-definition go-app --query taskDefinition > task-definition.json + aws ecs describe-task-definition --task-definition surepay-task --query taskDefinition > task-definition.json cat task-definition.json - name: Fill in the new image ID in the Amazon ECS task definition From db31828c0cfc8d9efef1b51edc0052a2446c0dab Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 06:38:59 +0100 Subject: [PATCH 141/148] Full Functional version e2e --- main.tf | 121 +++++++--------- modules/ecs/iam.tf | 25 ++++ modules/ecs/main.tf | 89 ++++++++++++ modules/ecs/network.tf | 65 +++++++++ modules/ecs/outputs.tf | 8 ++ modules/ecs/variables.tf | 34 +++++ modules/keypair/main.tf | 4 - modules/keypair/outputs.tf | 9 -- modules/keypair/variables.tf | 1 - modules/runner/data.tf | 15 ++ modules/runner/iam.tf | 81 ++++++++--- modules/runner/main.tf | 142 +++++++++++++++---- modules/runner/network.tf | 136 +++++++++++++++--- modules/runner/outputs.tf | 34 +++++ modules/runner/user_data.sh | 51 +++++++ modules/runner/userdata.sh | 24 ---- modules/runner/variables.tf | 246 ++++++++++++++++++++++++++++++++- modules/suplement/main.tf | 19 +++ modules/suplement/outputs.tf | 19 +++ modules/suplement/variables.tf | 4 + modules/vpc/main.tf | 0 variables.tf | 15 +- 22 files changed, 951 insertions(+), 191 deletions(-) create mode 100644 modules/ecs/iam.tf create mode 100644 modules/ecs/main.tf create mode 100644 modules/ecs/network.tf create mode 100644 modules/ecs/outputs.tf create mode 100644 modules/ecs/variables.tf delete mode 100644 modules/keypair/main.tf delete mode 100644 modules/keypair/outputs.tf delete mode 100644 modules/keypair/variables.tf create mode 100644 modules/runner/data.tf create mode 100644 modules/runner/outputs.tf create mode 100644 modules/runner/user_data.sh delete mode 100644 modules/runner/userdata.sh create mode 100644 modules/suplement/main.tf create mode 100644 modules/suplement/outputs.tf create mode 100644 modules/suplement/variables.tf delete mode 100644 modules/vpc/main.tf diff --git a/main.tf b/main.tf index 71a05fe..b7e95d8 100644 --- a/main.tf +++ b/main.tf @@ -19,8 +19,6 @@ module "vpc" { manage_default_route_table = false manage_default_security_group = false - - default_vpc_tags = merge( var.default_tags, { Name = "vpc" @@ -28,81 +26,58 @@ module "vpc" { ) } -# module "runner" { -# source = "./modules/runner" - -# vpc_id = module.vpc.vpc_id -# subnet_id = tostring(module.vpc.public_subnets[0]) -# key_pair_name = module.keypair.key_pair_name - -# ami_id = var.ami_id -# runner_instance_type = "t2.micro" - -# default_tags = merge( -# var.default_tags, { -# Name = "gh-runner" -# }, -# ) -# } - -# module "eks" { -# source = "terraform-aws-modules/eks/aws" -# version = "~> 20.0" - -# cluster_name = "${var.project}-al2023" -# cluster_version = "1.31" - -# # EKS Addons -# cluster_addons = { -# coredns = {} -# eks-pod-identity-agent = {} -# kube-proxy = {} -# vpc-cni = {} -# } - -# vpc_id = module.vpc.vpc_id -# subnet_ids = module.vpc.public_subnets - -# eks_managed_node_groups = { -# main = { -# # Starting on 1.30, AL2023 is the default AMI type for EKS managed node groups -# instance_types = [var.instance_type] - -# min_size = 1 -# max_size = 3 -# # This value is ignored after the initial creation -# # https://github.com/bryantbiggs/eks-desired-size-hack -# desired_size = 1 -# } -# } - -# tags = merge( -# var.default_tags, { -# Name = "eks" -# }, -# ) -# } - -module "bastion" { - source = "Guimove/bastion/aws" - bastion_ami = var.ami_id - bucket_name = "${var.project}-bastion-bucket" - region = var.region - vpc_id = module.vpc.vpc_id - instance_type = var.instance_type +module "runner" { + source = "./modules/runner" + runner_ami = var.ami_id + region = var.region + + instance_type = var.instance_type + + aws_ecs_service_name = module.ecs.service_name + aws_ecs_cluster_name = module.ecs.cluster_name is_lb_private = "false" - bastion_host_key_pair = module.keypair.key_pair_name + runner_host_key_pair = module.suplement.key_pair_name + runner_iam_policy_name = "${var.project}RunnerHostPolicy" create_dns_record = "false" - bastion_iam_policy_name = "${var.project}BastionHostPolicy" - elb_subnets = [module.vpc.public_subnets[0], module.vpc.public_subnets[1]] - auto_scaling_group_subnets = [module.vpc.public_subnets[0], module.vpc.public_subnets[1]] + vpc_id = module.vpc.vpc_id + elb_subnets = module.vpc.public_subnets + auto_scaling_group_subnets = module.vpc.private_subnets + ipv4_cidr_block = flatten([module.vpc.private_subnets_cidr_blocks, "0.0.0.0/0"]) + + github_url = "https://github.com/dilsilva/surepay/settings/actions/runners" + github_owner = "dilsilva" + github_repo = "surepay" + ssm_parameter_name = module.suplement.ssm_parameter_name + github_runner_group = "" + github_runner_labels = "" + tags = merge( var.default_tags, { - "name" = "${var.project}-bastion" + "name" = "${var.project}-runner" }, ) + + depends_on = [module.vpc] } -module "keypair" { - source = "./modules/keypair" - project = var.project -} \ No newline at end of file + +module "ecs" { + source = "./modules/ecs" + + project = var.project + region = var.region + vpc_id = module.vpc.vpc_id + public_subnets = module.vpc.public_subnets + private_subnets = module.vpc.private_subnets + app_name = var.app_name + app_image = module.suplement.ecr_repository + + depends_on = [module.vpc] +} + +module "suplement" { + source = "./modules/suplement" + project = var.project + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDJaDTgLeCoQqvOD2Hbityz0WD+/jmLY7Wpy4Zmu81S2ejbSiU2AHAZil0KhCJzMDtoZfhbsntRCU1i5tIVnCyT1XOptMXrMn3h8LqVHjc7KqMZkOnPHFjUm/JBgAsxyM4NOVIgLykH4QRotRCBtMhjPWwDfpdgrlFciEmq6NEiyVkNRWT2RJ2FV9JqD15vs9i3Q/whmR6nbqb7o5HCPRz6s2wkQonsjP16v+MpPZjFswGMJxcsL4ZcKN4bvsElhwYVGDSS1R6Z4cn/CSU8bluRPIHWUSEZsW9vME7h32j2v79qBp5I8ACJbyQC2VstoHRWSOoVt/sQE3gLjGBd+goi7sQCHDVQnhstSPuxZOdEuxGDANSEyyo7TCiZrfRVZqcDtmUi1WmTkAzpvFjQYZT8hwIxsVbp2VG3tP6UwH3DH8ofxd6eIOvH27bxlbwzbOAkNG9/rwT4kGfyZZ2D8R9aH9PFXeeohiQkJegyRzzIWzHhxtL2v5i2Mxcbtnhj/kdzK0GUUymDjO3LK7+UW4kGEKCX/KxuuWWsrlrKPTMZu1x3nsDJD+gUgC33GOkY7zO0hSj4kXVxpPN+Q5RngNB9rHF7RPRMuS7TCF0V6ZfTRh9Q6DNDrGOrzlLmJj2yA0vB/V2rsLRA/TXVpTlE91/j/1vxsIFuZ99NspCUwbBABw== dilsilva.diego@gmail.com" + pat_value = "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" + app_name = var.app_name +} diff --git a/modules/ecs/iam.tf b/modules/ecs/iam.tf new file mode 100644 index 0000000..13a55d3 --- /dev/null +++ b/modules/ecs/iam.tf @@ -0,0 +1,25 @@ +resource "aws_iam_role" "task-execution-role" { + name = "ecsTaskExecutionRole" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = [ + "ecs.amazonaws.com", + "lambda.amazonaws.com", + "ecs-tasks.amazonaws.com" + ] + } + }, + ] + }) +} + +resource "aws_iam_role_policy_attachment" "policy-attach" { + role = aws_iam_role.task-execution-role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} diff --git a/modules/ecs/main.tf b/modules/ecs/main.tf new file mode 100644 index 0000000..053e1bc --- /dev/null +++ b/modules/ecs/main.tf @@ -0,0 +1,89 @@ + + +resource "aws_ecs_cluster" "main" { + name = "${var.project}-ecs" + + setting { + name = "containerInsights" + value = "enabled" + } +} + +resource "aws_ecs_cluster_capacity_providers" "ecs_capacity_provider" { + cluster_name = aws_ecs_cluster.main.name + + capacity_providers = ["FARGATE"] + + default_capacity_provider_strategy { + base = 1 + weight = 100 + capacity_provider = "FARGATE" + } +} + +resource "aws_ecs_service" "service" { + name = "${var.project}-service" + cluster = aws_ecs_cluster.main.id + task_definition = aws_ecs_task_definition.task.arn + desired_count = 3 + + + force_new_deployment = true + + network_configuration { + subnets = [for subnet in var.private_subnets : subnet] + security_groups = [aws_security_group.ecs.id] + } + + triggers = { + redeployment = timestamp() + } + + load_balancer { + target_group_arn = aws_lb_target_group.ecs_tg.arn + container_name = var.app_name + container_port = 80 + } + + depends_on = [aws_lb.ecs_alb] +} + + +resource "aws_ecs_task_definition" "task" { + family = "${var.project}-task" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = 256 + memory = 512 + execution_role_arn = aws_iam_role.task-execution-role.arn + + container_definitions = jsonencode([ + { + name = var.app_name + image = var.app_image + cpu = 256 + memory = 512 + network_mode = "awsvpc" + essential = true + portMappings = [ + { + containerPort = 80 + hostPort = 80 + } + ] + logConfiguration = { + logDriver = "awslogs", + options = { + awslogs-group = "/ecs/${var.app_name}", + awslogs-region = var.region, + awslogs-stream-prefix = "ecs" + } + } + } + ]) + + ephemeral_storage { + size_in_gib = "50" + } + +} \ No newline at end of file diff --git a/modules/ecs/network.tf b/modules/ecs/network.tf new file mode 100644 index 0000000..f86fda5 --- /dev/null +++ b/modules/ecs/network.tf @@ -0,0 +1,65 @@ +resource "aws_security_group" "ecs" { + name = "${var.project}-ecs-security-group" + vpc_id = var.vpc_id + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + self = "false" + cidr_blocks = ["0.0.0.0/0"] + description = "any" + } + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + self = "false" + cidr_blocks = ["0.0.0.0/0"] + description = "any" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + + +resource "aws_lb" "ecs_alb" { + name = "${var.project}-ecs-alb" + internal = false + load_balancer_type = "application" + subnets = [for subnet in var.public_subnets : subnet] + security_groups = [aws_security_group.ecs.id] + + tags = { + Name = "ecs-alb" + } +} + +resource "aws_lb_listener" "ecs_alb_listener" { + load_balancer_arn = aws_lb.ecs_alb.arn + port = 80 + protocol = "HTTP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.ecs_tg.arn + } +} + +resource "aws_lb_target_group" "ecs_tg" { + name = "${var.project}-ecs-target-group" + port = 80 + protocol = "HTTP" + target_type = "ip" + vpc_id = var.vpc_id + + health_check { + path = "/" + } +} \ No newline at end of file diff --git a/modules/ecs/outputs.tf b/modules/ecs/outputs.tf new file mode 100644 index 0000000..05a58cd --- /dev/null +++ b/modules/ecs/outputs.tf @@ -0,0 +1,8 @@ +output "service_name" { + description = "Name of the created ECS service" + value = aws_ecs_service.service.name +} +output "cluster_name" { + description = "Name of the created ECS Cluster" + value = aws_ecs_cluster.main.name +} diff --git a/modules/ecs/variables.tf b/modules/ecs/variables.tf new file mode 100644 index 0000000..2c7d0be --- /dev/null +++ b/modules/ecs/variables.tf @@ -0,0 +1,34 @@ +variable "project" { + type = string + description = "Name of the project" + default = "surepay" +} +variable "region" { + type = string + description = "Region of the project" +} + +variable "vpc_id" { + type = string + description = "ID of the VPC" +} + +variable "public_subnets" { + type = list(string) + description = "Public subnets of the VPC" +} + +variable "private_subnets" { + type = list(string) + description = "Private subnets of the VPC" +} + +variable "app_name" { + type = string + description = "Name of the APP" +} + +variable "app_image" { + type = string + description = "Image of the APP" +} diff --git a/modules/keypair/main.tf b/modules/keypair/main.tf deleted file mode 100644 index 40b957c..0000000 --- a/modules/keypair/main.tf +++ /dev/null @@ -1,4 +0,0 @@ -resource "aws_key_pair" "keypair" { - key_name = "${var.project}-keypair" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDJaDTgLeCoQqvOD2Hbityz0WD+/jmLY7Wpy4Zmu81S2ejbSiU2AHAZil0KhCJzMDtoZfhbsntRCU1i5tIVnCyT1XOptMXrMn3h8LqVHjc7KqMZkOnPHFjUm/JBgAsxyM4NOVIgLykH4QRotRCBtMhjPWwDfpdgrlFciEmq6NEiyVkNRWT2RJ2FV9JqD15vs9i3Q/whmR6nbqb7o5HCPRz6s2wkQonsjP16v+MpPZjFswGMJxcsL4ZcKN4bvsElhwYVGDSS1R6Z4cn/CSU8bluRPIHWUSEZsW9vME7h32j2v79qBp5I8ACJbyQC2VstoHRWSOoVt/sQE3gLjGBd+goi7sQCHDVQnhstSPuxZOdEuxGDANSEyyo7TCiZrfRVZqcDtmUi1WmTkAzpvFjQYZT8hwIxsVbp2VG3tP6UwH3DH8ofxd6eIOvH27bxlbwzbOAkNG9/rwT4kGfyZZ2D8R9aH9PFXeeohiQkJegyRzzIWzHhxtL2v5i2Mxcbtnhj/kdzK0GUUymDjO3LK7+UW4kGEKCX/KxuuWWsrlrKPTMZu1x3nsDJD+gUgC33GOkY7zO0hSj4kXVxpPN+Q5RngNB9rHF7RPRMuS7TCF0V6ZfTRh9Q6DNDrGOrzlLmJj2yA0vB/V2rsLRA/TXVpTlE91/j/1vxsIFuZ99NspCUwbBABw== dilsilva.diego@gmail.com" -} \ No newline at end of file diff --git a/modules/keypair/outputs.tf b/modules/keypair/outputs.tf deleted file mode 100644 index bde488d..0000000 --- a/modules/keypair/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "key_pair_name" { - description = "The name of the key for the EC2" - value = aws_key_pair.keypair.key_name -} - -output "key_pair_arn" { - description = "The ARN of the key for the EC2" - value = aws_key_pair.keypair.arn -} \ No newline at end of file diff --git a/modules/keypair/variables.tf b/modules/keypair/variables.tf deleted file mode 100644 index eeec148..0000000 --- a/modules/keypair/variables.tf +++ /dev/null @@ -1 +0,0 @@ -variable "project" {} diff --git a/modules/runner/data.tf b/modules/runner/data.tf new file mode 100644 index 0000000..9b3f6cd --- /dev/null +++ b/modules/runner/data.tf @@ -0,0 +1,15 @@ +# Get latest Ubuntu Linux Distro 24.04 AMI +data "aws_ami" "ubuntu-linux-2404" { + most_recent = true + owners = ["099720109477"] # Canonical + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20241109*"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +data "aws_caller_identity" "current" {} \ No newline at end of file diff --git a/modules/runner/iam.tf b/modules/runner/iam.tf index 5917881..4f68049 100644 --- a/modules/runner/iam.tf +++ b/modules/runner/iam.tf @@ -1,41 +1,80 @@ -resource "aws_iam_instance_profile" "runner_instance_profile" { - name = "runner_instance_profile" - role = aws_iam_role.github_actions_assume_role.name -} -data "aws_iam_policy_document" "assume_role" { +data "aws_iam_policy_document" "assume_policy_document" { statement { - effect = "Allow" - + actions = [ + "sts:AssumeRole" + ] principals { type = "Service" identifiers = ["ec2.amazonaws.com"] } - - actions = ["sts:AssumeRole"] } } -# Apply Least Privilege ASAP -data "aws_iam_policy_document" "runner_policy" { +resource "aws_iam_role" "runner_host_role" { + name = var.runner_iam_role_name + path = "/" + assume_role_policy = data.aws_iam_policy_document.assume_policy_document.json +} + +data "aws_iam_policy_document" "runner_host_policy_document" { + statement { - actions = ["ec2:ReplaceIamInstanceProfileAssociation", "ec2:AssociateIamInstanceProfile"] + actions = [ + "ecs:RegisterTaskDefinition", + "ecs:DescribeTaskDefinition" + ] resources = ["*"] } + statement { - actions = ["iam:PassRole"] + actions = [ + "iam:PassRole" + ] + resources = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/ecsTaskExecutionRole"] + } + + statement { + actions = [ + "ecs:UpdateService", + "ecs:DescribeServices" + ] + resources = ["arn:aws:ecs:${var.region}:${data.aws_caller_identity.current.account_id}:service/${var.aws_ecs_cluster_name}/${var.aws_ecs_service_name}"] + } + statement { + actions = [ + + "ecr:CompleteLayerUpload", + "ecr:GetAuthorizationToken", + "ecr:UploadLayerPart", + "ecr:InitiateLayerUpload", + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage" + ] resources = ["*"] } + statement { + actions = [ + "ssm:GetParameters", + "ssm:GetParameter", + "ssm:GetParametersByPath" + ] + resources = ["arn:aws:ssm:${var.region}:${data.aws_caller_identity.current.account_id}:parameter/${var.ssm_parameter_name}"] + } + statement { + actions = [ + "ssm:DescribeParameters" + ] + resources = ["arn:aws:ssm:${var.region}:${data.aws_caller_identity.current.account_id}:parameter/${var.ssm_parameter_name}"] + } } -resource "aws_iam_role_policy" "runner_policy" { - name = "runner_policy" - role = aws_iam_role.github_actions_assume_role.id - policy = data.aws_iam_policy_document.runner_policy.json +resource "aws_iam_policy" "runner_host_policy" { + name = var.runner_iam_policy_name + policy = data.aws_iam_policy_document.runner_host_policy_document.json } -resource "aws_iam_role" "github_actions_assume_role" { - name = "github_actions_assume_role" - path = "/" - assume_role_policy = data.aws_iam_policy_document.assume_role.json +resource "aws_iam_role_policy_attachment" "runner_host" { + policy_arn = aws_iam_policy.runner_host_policy.arn + role = aws_iam_role.runner_host_role.name } \ No newline at end of file diff --git a/modules/runner/main.tf b/modules/runner/main.tf index bd3f420..9a389fe 100644 --- a/modules/runner/main.tf +++ b/modules/runner/main.tf @@ -1,37 +1,123 @@ -#Latest Spot instance -data "aws_ami" "amazonlinux" { - most_recent = true - owners = ["amazon"] - filter { - name = "image-id" - values = [var.ami_id] - } +locals { + security_group = join("", flatten([aws_security_group.runner_host_security_group[*].id, var.runner_security_group_id])) +} + + +resource "aws_iam_instance_profile" "runner_host_profile" { + role = aws_iam_role.runner_host_role.name + path = "/" } -resource "aws_instance" "gh-runner" { - ami = data.aws_ami.amazonlinux.id - instance_type = var.runner_instance_type - iam_instance_profile = aws_iam_instance_profile.runner_instance_profile.name - user_data = file("${path.module}/userdata.sh") +resource "aws_launch_template" "runner_launch_template" { + name_prefix = var.project + image_id = var.runner_ami != "" ? var.runner_ami : data.aws_ami.ubuntu-linux-2404.id + instance_type = var.instance_type + update_default_version = true + + monitoring { + enabled = true + } + + network_interfaces { + associate_public_ip_address = var.associate_public_ip_address + security_groups = concat([local.security_group], var.runner_additional_security_groups) + delete_on_termination = true + } + + iam_instance_profile { + name = aws_iam_instance_profile.runner_host_profile.name + } + + key_name = var.runner_host_key_pair + + user_data = base64encode(templatefile("${path.module}/user_data.sh", { + aws_region = var.region + github_url = var.github_url + github_owner = var.github_owner + github_repo = var.github_repo + ssm_parameter_name = var.ssm_parameter_name + runner_group = var.github_runner_group + runner_labels = var.github_runner_labels + })) + + block_device_mappings { + device_name = "/dev/sda1" + ebs { + volume_size = var.disk_size + volume_type = "gp2" + delete_on_termination = true + encrypted = var.disk_encrypt + } + } - subnet_id = var.subnet_id - vpc_security_group_ids = [aws_security_group.gh-runner.id] + tag_specifications { + resource_type = "instance" + tags = merge(tomap({ "Name" = var.runner_launch_template_name }), merge(var.tags)) + } - key_name = var.key_pair_name + tag_specifications { + resource_type = "volume" + tags = merge(tomap({ "Name" = var.runner_launch_template_name }), merge(var.tags)) + } - tags = merge( - var.default_tags, { - Name = "GH Self-hosted Runner" - }) + metadata_options { + http_endpoint = var.http_endpoint ? "enabled" : "disabled" + http_tokens = var.use_imds_v2 ? "required" : "optional" + http_put_response_hop_limit = var.http_put_response_hop_limit + instance_metadata_tags = var.enable_instance_metadata_tags ? "enabled" : "disabled" + } - root_block_device { - volume_type = "gp3" - volume_size = 50 - delete_on_termination = true + lifecycle { + create_before_destroy = true } } -resource "aws_key_pair" "surepay-keypair" { - key_name = "deployer-key" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDJaDTgLeCoQqvOD2Hbityz0WD+/jmLY7Wpy4Zmu81S2ejbSiU2AHAZil0KhCJzMDtoZfhbsntRCU1i5tIVnCyT1XOptMXrMn3h8LqVHjc7KqMZkOnPHFjUm/JBgAsxyM4NOVIgLykH4QRotRCBtMhjPWwDfpdgrlFciEmq6NEiyVkNRWT2RJ2FV9JqD15vs9i3Q/whmR6nbqb7o5HCPRz6s2wkQonsjP16v+MpPZjFswGMJxcsL4ZcKN4bvsElhwYVGDSS1R6Z4cn/CSU8bluRPIHWUSEZsW9vME7h32j2v79qBp5I8ACJbyQC2VstoHRWSOoVt/sQE3gLjGBd+goi7sQCHDVQnhstSPuxZOdEuxGDANSEyyo7TCiZrfRVZqcDtmUi1WmTkAzpvFjQYZT8hwIxsVbp2VG3tP6UwH3DH8ofxd6eIOvH27bxlbwzbOAkNG9/rwT4kGfyZZ2D8R9aH9PFXeeohiQkJegyRzzIWzHhxtL2v5i2Mxcbtnhj/kdzK0GUUymDjO3LK7+UW4kGEKCX/KxuuWWsrlrKPTMZu1x3nsDJD+gUgC33GOkY7zO0hSj4kXVxpPN+Q5RngNB9rHF7RPRMuS7TCF0V6ZfTRh9Q6DNDrGOrzlLmJj2yA0vB/V2rsLRA/TXVpTlE91/j/1vxsIFuZ99NspCUwbBABw== dilsilva.diego@gmail.com" -} \ No newline at end of file +resource "aws_autoscaling_group" "runner_auto_scaling_group" { + name_prefix = "ASG-${var.project}" + launch_template { + id = aws_launch_template.runner_launch_template.id + version = aws_launch_template.runner_launch_template.latest_version + } + max_size = var.runner_instance_count_max + min_size = var.runner_instance_count_min + desired_capacity = var.runner_instance_count + + vpc_zone_identifier = var.auto_scaling_group_subnets + + default_cooldown = 180 + health_check_grace_period = 180 + health_check_type = "EC2" + + target_group_arns = var.create_elb ? [ + aws_lb_target_group.runner_lb_target_group[0].arn, + ] : null + + termination_policies = [ + "OldestLaunchConfiguration", + ] + + dynamic "tag" { + for_each = var.tags + + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } + } + + tag { + key = "Name" + value = "ASG-${var.project}" + propagate_at_launch = true + } + + instance_refresh { + strategy = "Rolling" + } + + lifecycle { + create_before_destroy = true + } + +} diff --git a/modules/runner/network.tf b/modules/runner/network.tf index 72d03ec..05e35b7 100644 --- a/modules/runner/network.tf +++ b/modules/runner/network.tf @@ -1,27 +1,125 @@ -resource "aws_security_group" "gh-runner" { - name = "gh-runner" - description = "Allows necessary ports to gh runner communicate" + +resource "aws_route53_record" "runner_record_name" { + name = var.runner_record_name + zone_id = var.hosted_zone_id + type = "A" + count = var.create_dns_record && var.create_elb ? 1 : 0 + + alias { + evaluate_target_health = true + name = aws_lb.runner_lb[0].dns_name + zone_id = aws_lb.runner_lb[0].zone_id + } +} + +resource "aws_lb" "runner_lb" { + count = var.create_elb ? 1 : 0 + + internal = var.is_lb_private + name = "${var.project}-lb" + + subnets = var.elb_subnets + + load_balancer_type = "network" + tags = merge(var.tags) + + lifecycle { + precondition { + condition = !var.create_elb || (length(var.elb_subnets) > 0 && var.is_lb_private != null) + error_message = "elb_subnets and is_lb_private must be set when creating a load balancer" + } + } +} + +resource "aws_lb_target_group" "runner_lb_target_group" { + count = var.create_elb ? 1 : 0 + + name = "${var.project}-lb-target" + port = var.public_ssh_port + protocol = "TCP" + vpc_id = var.vpc_id + target_type = "instance" + + health_check { + port = "traffic-port" + protocol = "TCP" + } + + tags = merge(var.tags) +} + +resource "aws_lb_listener" "runner_lb_listener_22" { + count = var.create_elb ? 1 : 0 + + default_action { + target_group_arn = aws_lb_target_group.runner_lb_target_group[0].arn + type = "forward" + } + + load_balancer_arn = aws_lb.runner_lb[0].arn + port = var.public_ssh_port + protocol = "TCP" +} + +#SGs +resource "aws_security_group" "runner_host_security_group" { + count = var.runner_security_group_id == "" ? 1 : 0 + description = "Enable SSH access to the runner host from external via SSH port" + name = "${var.project}-host" vpc_id = var.vpc_id + + tags = merge(var.tags) } -resource "aws_vpc_security_group_ingress_rule" "allow_tls" { - security_group_id = aws_security_group.gh-runner.id - cidr_ipv4 = "0.0.0.0/0" - from_port = 443 - ip_protocol = "tcp" - to_port = 443 + +resource "aws_security_group" "lb" { + description = "Enable SSH access to LB via SSH port" + name = "${var.project}-ssh-lb" + vpc_id = var.vpc_id + + tags = merge(var.tags) } -resource "aws_vpc_security_group_ingress_rule" "allow_ssh" { - security_group_id = aws_security_group.gh-runner.id - cidr_ipv4 = "0.0.0.0/0" - from_port = 22 - ip_protocol = "tcp" - to_port = 22 +resource "aws_security_group_rule" "ingress_runner" { + count = var.runner_security_group_id == "" && var.create_elb ? 1 : 0 + description = "Incoming traffic to runner" + type = "ingress" + from_port = var.public_ssh_port + to_port = var.public_ssh_port + protocol = "TCP" + cidr_blocks = var.ipv4_cidr_block + + security_group_id = local.security_group } -resource "aws_vpc_security_group_egress_rule" "allow_all_traffic" { - security_group_id = aws_security_group.gh-runner.id - cidr_ipv4 = "0.0.0.0/0" - ip_protocol = "-1" # semantically equivalent to all ports +resource "aws_security_group_rule" "egress_runner" { + count = var.runner_security_group_id == "" ? 1 : 0 + description = "Outgoing traffic from runner to instances" + type = "egress" + from_port = "0" + to_port = "65535" + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = local.security_group +} + +resource "aws_security_group" "private_instances_security_group" { + description = "Enable SSH access to the Private instances from the runner via SSH port" + name = "${var.project}-priv-instances" + vpc_id = var.vpc_id + + tags = merge(var.tags) +} + +resource "aws_security_group_rule" "ingress_instances" { + description = "Incoming traffic from runner" + type = "ingress" + from_port = var.private_ssh_port + to_port = var.private_ssh_port + protocol = "TCP" + + source_security_group_id = local.security_group + + security_group_id = aws_security_group.private_instances_security_group.id } \ No newline at end of file diff --git a/modules/runner/outputs.tf b/modules/runner/outputs.tf new file mode 100644 index 0000000..3aaae51 --- /dev/null +++ b/modules/runner/outputs.tf @@ -0,0 +1,34 @@ +output "runner_auto_scaling_group_name" { + description = "The name of the Auto Scaling Group for runner hosts" + value = aws_autoscaling_group.runner_auto_scaling_group.name +} + +output "runner_elb_id" { + description = "The ID of the ELB for runner hosts" + value = var.create_elb ? try(aws_lb.runner_lb[0].id, null) : null +} + +output "runner_host_security_group" { + description = "The ID of the runner host security group" + value = aws_security_group.runner_host_security_group[*].id +} + +output "elb_arn" { + description = "The ARN of the ELB for runner hosts" + value = var.create_elb ? try(aws_lb.runner_lb[0].arn, null) : null +} + +output "elb_ip" { + description = "The DNS name of the ELB for runner hosts" + value = var.create_elb ? try(aws_lb.runner_lb[0].dns_name, null) : null +} + +output "private_instances_security_group" { + description = "The ID of the security group for private instances" + value = aws_security_group.private_instances_security_group.id +} + +output "target_group_arn" { + description = "The ARN of the target group for the ELB" + value = var.create_elb ? try(aws_lb_target_group.runner_lb_target_group[0].arn, null) : null +} diff --git a/modules/runner/user_data.sh b/modules/runner/user_data.sh new file mode 100644 index 0000000..d6bdc18 --- /dev/null +++ b/modules/runner/user_data.sh @@ -0,0 +1,51 @@ +#!/bin/bash -x +sudo apt-get update +sudo apt install jq git unzip ca-certificates curl -y + +#Docker +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ +sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli -y +sudo groupadd docker +sudo usermod -aG docker ubuntu +sudo newgrp docker + + +# Golang +sudo wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz +sudo tar -C /usr/local -xzf go1.23.4.linux-amd64.tar.gz +export PATH=$PATH:/usr/local/go/bin + + +# awscli +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install +sudo rm -rf awscliv2.zip + +mkdir /home/ubuntu/actions-runner && cd /home/ubuntu/actions-runner + +curl -s https://github.com/actions/runner/releases | grep -o -E "https://github.*actions-runner-linux-x64-[0-9\.]+.tar.gz" | sort | uniq > versions.txt +RUNNER_FILE_LINK=`cat versions.txt | tail -n1` +curl -o actions-runner-linux-x64.tar.gz -L $RUNNER_FILE_LINK +tar xzf ./actions-runner-linux-x64.tar.gz + +# Configure actions-runner +PERSONAL_ACCESS_TOKEN=`aws ssm get-parameter --with-decryption --name ${ssm_parameter_name} --region ${aws_region} | jq -r '.Parameter.Value'` +TOKEN_RESPONSE=`curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: token $PERSONAL_ACCESS_TOKEN" https://api.github.com/repos/${github_owner}/${github_repo}/actions/runners/registration-token` +TOKEN=`echo $TOKEN_RESPONSE | jq -r '.token'` +RUNNER_ALLOW_RUNASROOT=true ./config.sh --url ${github_url} --unattended --token $TOKEN ${runner_group} ${runner_labels} + +# Install as service +RUNNER_ALLOW_RUNASROOT=true ./svc.sh install +echo ==== ACTIONS-RUNNER DONE ==== +INSTANCE_ID=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id` +aws ec2 create-tags --region ${aws_region} --resources $INSTANCE_ID --tags Key=terraform-aws-github-runner:setup,Value=done + +RUNNER_ALLOW_RUNASROOT=true ./run.sh \ No newline at end of file diff --git a/modules/runner/userdata.sh b/modules/runner/userdata.sh deleted file mode 100644 index 2ba5084..0000000 --- a/modules/runner/userdata.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -#Github Actions -sudo yum update -y && \ -sudo yum install docker -y && \ -sudo yum install git -y && \ -sudo yum install libicu -y && \ -sudo systemctl enable docker - -# Create a folder -mkdir actions-runner && cd actions-runner -# Download the latest runner package -curl -o actions-runner-linux-x64-2.321.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.321.0/actions-runner-linux-x64-2.321.0.tar.gz -# Optional: Validate the hash -echo "ba46ba7ce3a4d7236b16fbe44419fb453bc08f866b24f04d549ec89f1722a29e actions-runner-linux-x64-2.321.0.tar.gz" | shasum -a 256 -c -# Extract the installer -tar xzf ./actions-runner-linux-x64-2.321.0.tar.gz -# Create the runner and start the configuration experience -./config.sh --unattended --url https://github.com/dilsilva/surepay --token ADMDQVGIQVD3IP7QSANPRQLHJUHQE -# Last step, run it! -./run.sh& - -# Setup as system service -./svc.sh \ No newline at end of file diff --git a/modules/runner/variables.tf b/modules/runner/variables.tf index b8cf7e7..ec96c5a 100644 --- a/modules/runner/variables.tf +++ b/modules/runner/variables.tf @@ -1,6 +1,240 @@ -variable "runner_instance_type" {} -variable "default_tags" {} -variable "subnet_id" {} -variable "ami_id" {} -variable "vpc_id" {} -variable "key_pair_name" {} \ No newline at end of file +variable "project" { + type = string + description = "Name of the project" + default = "surepay" +} + +variable "associate_public_ip_address" { + type = bool + default = true +} + +variable "auto_scaling_group_subnets" { + type = list(string) + description = "List of subnets where the Auto Scaling Group will deploy the instances" +} + +variable "runner_additional_security_groups" { + type = list(string) + description = "List of additional security groups to attach to the launch template" + default = [] +} + +variable "runner_ami" { + type = string + description = "The AMI that the runner Host will use." + default = "" +} + +variable "runner_host_key_pair" { + type = string + description = "Select the key pair to use to launch the runner host" +} + +variable "runner_iam_policy_name" { + type = string + description = "IAM policy name to create for granting the instance role access to the bucket" + default = "runnerHost" +} + +variable "runner_iam_role_name" { + type = string + description = "IAM role name to create" + default = "runnerRole" +} + +variable "runner_instance_count" { + type = number + default = 1 +} + +variable "runner_instance_count_min" { + type = number + default = 1 +} + +variable "runner_instance_count_max" { + type = number + default = 3 +} + +variable "runner_launch_template_name" { + type = string + description = "runner Launch template Name, will also be used for the ASG" + default = "runner-lt" +} + +variable "runner_record_name" { + type = string + description = "DNS record name to use for the runner" + default = "" +} + +variable "runner_security_group_id" { + type = string + description = "Custom security group to use" + default = "" +} + +variable "cidrs" { + type = list(string) + description = "List of CIDRs that can access the runner. Default: 0.0.0.0/0" + + default = [ + "0.0.0.0/0", + ] +} + +variable "create_dns_record" { + type = bool + description = "Choose if you want to create a record name for the runner (LB). If true, 'hosted_zone_id' and 'runner_record_name' are mandatory" +} + +variable "create_elb" { + type = bool + description = "Choose if you want to deploy an ELB for accessing runner hosts. If true, you must set elb_subnets and is_lb_private" + default = true +} + +variable "disk_encrypt" { + type = bool + description = "Instance EBS encryption" + default = true +} + +variable "disk_size" { + type = number + description = "Root EBS size in GB" + default = 100 +} + +variable "elb_subnets" { + type = list(string) + description = "List of subnets where the ELB will be deployed" + default = [] +} + +variable "enable_instance_metadata_tags" { + type = bool + description = "Enables or disables access to instance tags from the instance metadata service" + default = false +} + +variable "hosted_zone_id" { + type = string + description = "Name of the hosted zone where we'll register the runner DNS name" + default = "" +} + +variable "http_endpoint" { + type = bool + description = "Whether the metadata service is available" + default = true +} + +variable "http_put_response_hop_limit" { + type = number + description = "The desired HTTP PUT response hop limit for instance metadata requests" + default = 1 +} + +variable "instance_type" { + type = string + description = "Instance size of the runner" + default = "t3.nano" +} + + +variable "is_lb_private" { + type = bool + nullable = true + default = null + description = "If TRUE, the load balancer scheme will be \"internal\" else \"internet-facing\"" +} + +variable "private_ssh_port" { + type = number + description = "Set the SSH port to use between the runner and private instance" + default = 22 +} + +variable "public_ssh_port" { + type = number + description = "Set the SSH port to use from desktop to the runner" + default = 22 +} + +variable "region" { + type = string +} + +variable "tags" { + type = map(string) + default = {} + description = "A mapping of tags to assign" +} + +variable "use_imds_v2" { + type = bool + description = "Use (IMDSv2) Instance Metadata Service V2" + default = false +} + +variable "vpc_id" { + type = string + description = "VPC ID where we'll deploy the runner" +} + +variable "ipv4_cidr_block" { + type = list(string) + default = [""] + description = "List of ipv4 CIDR blocks from the subnet" +} + +variable "github_url" { + type = string + default = "" + description = "GitHub full URL.
Example: \"https://github.com/cloudandthings/repo\"." +} + +variable "github_owner" { + type = string + default = "" + description = "GitHub repository owner." +} + +variable "github_repo" { + type = string + default = "" + description = "GitHub repository name." +} + +variable "ssm_parameter_name" { + type = string + default = "" + description = "SSM parameter name for the GitHub Runner token.
Example: \"/github/runner/token\"." +} + +variable "github_runner_group" { + type = string + default = "" + description = "Custom GitHub runner group." +} + +variable "github_runner_labels" { + type = string + default = "" + description = "Custom GitHub runner labels.
Example: `\"gpu,x64,linux\"`." +} + +variable "aws_ecs_cluster_name" { + type = string + default = "" + description = "Name of the ECS Cluster where the operations gonna be executed" +} + +variable "aws_ecs_service_name" { + type = string + default = "" + description = "Name of the ECS service where the operations gonna be executed" +} \ No newline at end of file diff --git a/modules/suplement/main.tf b/modules/suplement/main.tf new file mode 100644 index 0000000..d94b5af --- /dev/null +++ b/modules/suplement/main.tf @@ -0,0 +1,19 @@ +resource "aws_key_pair" "keypair" { + key_name = "${var.project}-keypair" + public_key = var.public_key +} + +resource "aws_ssm_parameter" "pat" { + name = "PERSONAL_ACCESS_TOKEN" + type = "String" + value = var.pat_value +} + +resource "aws_ecr_repository" "ecr" { + name = var.app_name + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = true + } +} \ No newline at end of file diff --git a/modules/suplement/outputs.tf b/modules/suplement/outputs.tf new file mode 100644 index 0000000..f4ff207 --- /dev/null +++ b/modules/suplement/outputs.tf @@ -0,0 +1,19 @@ +output "key_pair_name" { + description = "The name of the key for the EC2" + value = aws_key_pair.keypair.key_name +} + +output "key_pair_arn" { + description = "The ARN of the key for the EC2" + value = aws_key_pair.keypair.arn +} + +output "ssm_parameter_name" { + description = "The value of the parameter." + value = aws_ssm_parameter.pat.name +} + +output "ecr_repository" { + description = " The URL of the repository (in the form aws_account_id.dkr.ecr.region.amazonaws.com/repositoryName)." + value = aws_ecr_repository.ecr.repository_url +} \ No newline at end of file diff --git a/modules/suplement/variables.tf b/modules/suplement/variables.tf new file mode 100644 index 0000000..9abaf49 --- /dev/null +++ b/modules/suplement/variables.tf @@ -0,0 +1,4 @@ +variable "project" {} +variable "public_key" {} +variable "pat_value" {} +variable "app_name" {} diff --git a/modules/vpc/main.tf b/modules/vpc/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/variables.tf b/variables.tf index dcfdca9..16d7d12 100644 --- a/variables.tf +++ b/variables.tf @@ -12,8 +12,8 @@ variable "default_tags" { default = { Environment = "Dev" - Owner = "Surepay" - Project = "Surepay" + Owner = "surepay" + Project = "surepay" } description = "Default Tags for Project" @@ -45,11 +45,14 @@ variable "azs" { #Runner variable "ami_id" { default = "ami-0e9085e60087ce171" #Ubuntu - # default = "ami-047bb4163c506cd98" #Anazon Linyx - type = string + type = string } -#EKS variable "instance_type" { default = "t2.medium" -} \ No newline at end of file +} + +variable "app_name" { + default = "go-app" +} + From 9380d53939f6123ac85b411f8b926a025ca1348e Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 16:45:02 +0100 Subject: [PATCH 142/148] testing pipeline --- .github/workflows/runner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 52b848f..1256b0b 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -71,4 +71,4 @@ jobs: service: surepay-service cluster: surepay-ecs desired-count: 4 - wait-for-service-stability: true \ No newline at end of file + wait-for-service-stability: true From 99bb419468476a8bb5569f99b69855024df5ae87 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 19:25:28 +0100 Subject: [PATCH 143/148] testing pipe --- .github/workflows/runner.yaml | 8 +-- .github/workflows/tf.yaml | 41 ++++++++++++++ README.md | 41 ++++++++++++++ main.tf | 4 +- modules/README.md | 25 +++++++++ modules/ecs/README.md | 55 +++++++++++++++++++ modules/ecs/network.tf | 56 ++++++++++++++++++- modules/ecs/variables.tf | 5 ++ modules/runner/README.md | 100 ++++++++++++++++++++++++++++++++++ modules/runner/main.tf | 2 +- modules/runner/network.tf | 71 ++++++++++++++++-------- modules/runner/user_data.sh | 2 - modules/runner/variables.tf | 2 +- modules/suplement/README.md | 41 ++++++++++++++ variables.tf | 2 +- 15 files changed, 418 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/tf.yaml create mode 100644 README.md create mode 100644 modules/README.md create mode 100644 modules/ecs/README.md create mode 100644 modules/runner/README.md create mode 100644 modules/suplement/README.md diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index 1256b0b..b82bcd4 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,11 +1,6 @@ name: Deploy to ECR on: push -env: - AWS_ACCOUNT: 061051224299 - AWS_REGION: eu-west-1 - ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com - ECR_REPOSITORY: go-app jobs: build: @@ -18,6 +13,8 @@ jobs: - name: Build and tag image id: build-image env: + ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com + ECR_REPOSITORY: go-app TAG_COMMIT: ${{ github.sha }} run: | cd app/ @@ -40,6 +37,7 @@ jobs: env: AWS_ACCOUNT: 061051224299 AWS_REGION: eu-west-1 + ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com run: | docker login -u AWS -p $(aws ecr get-login-password --region $AWS_REGION) $ECR_REGISTRY diff --git a/.github/workflows/tf.yaml b/.github/workflows/tf.yaml new file mode 100644 index 0000000..c514bbd --- /dev/null +++ b/.github/workflows/tf.yaml @@ -0,0 +1,41 @@ +name: Terraform pipeline + docs +on: + - pull_request + +jobs: + tf-pipe: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - uses: actions/checkout@v4 + - uses: hashicorp/setup-terraform@v3 + + - name: Terraform fmt + id: fmt + run: terraform fmt -check + continue-on-error: true + + - name: Terraform Init + id: init + run: terraform init + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + + - name: Terraform Plan + id: plan + run: terraform plan -no-color + continue-on-error: true + + - name: Render terraform docs and push changes back to PR + uses: terraform-docs/gh-actions@main + with: + working-dir: . + output-file: README.md + output-method: inject + git-push: "true" + recursive: true diff --git a/README.md b/README.md new file mode 100644 index 0000000..76682f2 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [ecs](#module\_ecs) | ./modules/ecs | n/a | +| [runner](#module\_runner) | ./modules/runner | n/a | +| [suplement](#module\_suplement) | ./modules/suplement | n/a | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ami\_id](#input\_ami\_id) | Runner | `string` | `"ami-0e9085e60087ce171"` | no | +| [app\_name](#input\_app\_name) | n/a | `string` | `"go-app"` | no | +| [azs](#input\_azs) | n/a | `list(string)` | 
[
  "eu-west-1a",
  "eu-west-1b"
]
| no | +| [default\_tags](#input\_default\_tags) | Default Tags for Project | `map(string)` | 
{
  "Environment": "Dev",
  "Owner": "surepay",
  "Project": "surepay"
}
| no | +| [instance\_type](#input\_instance\_type) | n/a | `string` | `"t2.medium"` | no | +| [private\_subnets](#input\_private\_subnets) | n/a | `list(string)` | 
[
  "10.0.1.0/24",
  "10.0.2.0/24"
]
| no | +| [project](#input\_project) | n/a | `string` | `"surepay"` | no | +| [public\_subnets](#input\_public\_subnets) | n/a | `list(string)` | 
[
  "10.0.101.0/24",
  "10.0.102.0/24"
]
| no | +| [region](#input\_region) | n/a | `string` | `"eu-west-1"` | no | +| [vpc\_cidr](#input\_vpc\_cidr) | VPC | `string` | `"10.0.0.0/16"` | no | + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/main.tf b/main.tf index b7e95d8..1d14faf 100644 --- a/main.tf +++ b/main.tf @@ -56,7 +56,7 @@ module "runner" { "name" = "${var.project}-runner" }, ) - depends_on = [module.vpc] + depends_on = [module.vpc, module.suplement] } @@ -71,6 +71,8 @@ module "ecs" { app_name = var.app_name app_image = module.suplement.ecr_repository + private_instances_security_group = module.runner.private_instances_security_group + depends_on = [module.vpc] } diff --git a/modules/README.md b/modules/README.md new file mode 100644 index 0000000..ef2fa69 --- /dev/null +++ b/modules/README.md @@ -0,0 +1,25 @@ + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +No inputs. + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/modules/ecs/README.md b/modules/ecs/README.md new file mode 100644 index 0000000..09ac3c1 --- /dev/null +++ b/modules/ecs/README.md @@ -0,0 +1,55 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_ecs_cluster.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster) | resource | +| [aws_ecs_cluster_capacity_providers.ecs_capacity_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster_capacity_providers) | resource | +| [aws_ecs_service.service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource | +| [aws_ecs_task_definition.task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | +| [aws_iam_role.task-execution-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.policy-attach](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lb.ecs_alb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource | +| [aws_lb_listener.ecs_alb_listener](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource | +| [aws_lb_target_group.ecs_tg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_target_group) | resource | +| [aws_security_group.ecs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.ecs_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.egress_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.lb_runner_ssh](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.lb_runner_ssl](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.lb_runner_tcp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [app\_image](#input\_app\_image) | Image of the APP | `string` | n/a | yes | +| [app\_name](#input\_app\_name) | Name of the APP | `string` | n/a | yes | +| [private\_instances\_security\_group](#input\_private\_instances\_security\_group) | ID of the SG used for internal communication between instances | `string` | n/a | yes | +| [private\_subnets](#input\_private\_subnets) | Private subnets of the VPC | `list(string)` | n/a | yes | +| [project](#input\_project) | Name of the project | `string` | `"surepay"` | no | +| [public\_subnets](#input\_public\_subnets) | Public subnets of the VPC | `list(string)` | n/a | yes | +| [region](#input\_region) | Region of the project | `string` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | ID of the VPC | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_name](#output\_cluster\_name) | Name of the created ECS Cluster | +| [service\_name](#output\_service\_name) | Name of the created ECS service | + \ No newline at end of file diff --git a/modules/ecs/network.tf b/modules/ecs/network.tf index f86fda5..a102e5d 100644 --- a/modules/ecs/network.tf +++ b/modules/ecs/network.tf @@ -1,5 +1,5 @@ -resource "aws_security_group" "ecs" { - name = "${var.project}-ecs-security-group" +resource "aws_security_group" "ecs_lb" { + name = "${var.project}-ecs-lb-sg" vpc_id = var.vpc_id ingress { @@ -28,6 +28,58 @@ resource "aws_security_group" "ecs" { } } +resource "aws_security_group" "ecs" { + name = "${var.project}-ecs-security-group" + vpc_id = var.vpc_id + +} + +resource "aws_security_group_rule" "lb_runner_tcp" { + description = "Incoming shh traffic to ecs" + type = "ingress" + from_port = 22 + to_port = 22 + protocol = "TCP" + + security_group_id = aws_security_group.ecs.id + source_security_group_id = var.private_instances_security_group == "" ? aws_security_group.ecs.id : var.private_instances_security_group +} + +resource "aws_security_group_rule" "lb_runner_ssh" { + description = "Incoming tcp traffic to ecs" + type = "ingress" + from_port = 80 + to_port = 80 + protocol = "TCP" + + security_group_id = aws_security_group.ecs.id + source_security_group_id = aws_security_group.ecs_lb.id +} + +resource "aws_security_group_rule" "lb_runner_ssl" { + description = "Incoming ssl traffic to ecs" + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "TCP" + + security_group_id = aws_security_group.ecs.id + source_security_group_id = aws_security_group.ecs_lb.id +} + +resource "aws_security_group_rule" "egress_runner" { + description = "Outgoing traffic from runner to instances" + type = "egress" + from_port = "0" + to_port = "65535" + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.ecs.id +} + + +#LB resource "aws_lb" "ecs_alb" { name = "${var.project}-ecs-alb" diff --git a/modules/ecs/variables.tf b/modules/ecs/variables.tf index 2c7d0be..0569334 100644 --- a/modules/ecs/variables.tf +++ b/modules/ecs/variables.tf @@ -32,3 +32,8 @@ variable "app_image" { type = string description = "Image of the APP" } + +variable "private_instances_security_group" { + type = string + description = "ID of the SG used for internal communication between instances" +} diff --git a/modules/runner/README.md b/modules/runner/README.md new file mode 100644 index 0000000..b60e366 --- /dev/null +++ b/modules/runner/README.md @@ -0,0 +1,100 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_group.runner_auto_scaling_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | +| [aws_iam_instance_profile.runner_host_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.runner_host_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.runner_host_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.runner_host](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_launch_template.runner_launch_template](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_lb.runner_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource | +| [aws_lb_listener.runner_lb_listener_22](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource | +| [aws_lb_target_group.runner_lb_target_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_target_group) | resource | +| [aws_route53_record.runner_record_name](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_record) | resource | +| [aws_security_group.lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.private_instances_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.runner_host_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.all_egress_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.egress_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.ingress_instances](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.lb_runner_ssh](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.shh_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_ami.ubuntu-linux-2404](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.assume_policy_document](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.runner_host_policy_document](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | n/a | `bool` | `false` | no | +| [auto\_scaling\_group\_subnets](#input\_auto\_scaling\_group\_subnets) | List of subnets where the Auto Scaling Group will deploy the instances | `list(string)` | n/a | yes | +| [aws\_ecs\_cluster\_name](#input\_aws\_ecs\_cluster\_name) | Name of the ECS Cluster where the operations gonna be executed | `string` | `""` | no | +| [aws\_ecs\_service\_name](#input\_aws\_ecs\_service\_name) | Name of the ECS service where the operations gonna be executed | `string` | `""` | no | +| [cidrs](#input\_cidrs) | List of CIDRs that can access the runner. Default: 0.0.0.0/0 | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | +| [create\_dns\_record](#input\_create\_dns\_record) | Choose if you want to create a record name for the runner (LB). If true, 'hosted\_zone\_id' and 'runner\_record\_name' are mandatory | `bool` | n/a | yes | +| [create\_elb](#input\_create\_elb) | Choose if you want to deploy an ELB for accessing runner hosts. If true, you must set elb\_subnets and is\_lb\_private | `bool` | `true` | no | +| [disk\_encrypt](#input\_disk\_encrypt) | Instance EBS encryption | `bool` | `true` | no | +| [disk\_size](#input\_disk\_size) | Root EBS size in GB | `number` | `100` | no | +| [elb\_subnets](#input\_elb\_subnets) | List of subnets where the ELB will be deployed | `list(string)` | `[]` | no | +| [enable\_instance\_metadata\_tags](#input\_enable\_instance\_metadata\_tags) | Enables or disables access to instance tags from the instance metadata service | `bool` | `false` | no | +| [github\_owner](#input\_github\_owner) | GitHub repository owner. | `string` | `""` | no | +| [github\_repo](#input\_github\_repo) | GitHub repository name. | `string` | `""` | no | +| [github\_runner\_group](#input\_github\_runner\_group) | Custom GitHub runner group. | `string` | `""` | no | +| [github\_runner\_labels](#input\_github\_runner\_labels) | Custom GitHub runner labels.
Example: `"gpu,x64,linux"`. | `string` | `""` | no | +| [github\_url](#input\_github\_url) | GitHub full URL.
Example: "https://github.com/cloudandthings/repo". | `string` | `""` | no | +| [hosted\_zone\_id](#input\_hosted\_zone\_id) | Name of the hosted zone where we'll register the runner DNS name | `string` | `""` | no | +| [http\_endpoint](#input\_http\_endpoint) | Whether the metadata service is available | `bool` | `true` | no | +| [http\_put\_response\_hop\_limit](#input\_http\_put\_response\_hop\_limit) | The desired HTTP PUT response hop limit for instance metadata requests | `number` | `1` | no | +| [instance\_type](#input\_instance\_type) | Instance size of the runner | `string` | `"t3.nano"` | no | +| [ipv4\_cidr\_block](#input\_ipv4\_cidr\_block) | List of ipv4 CIDR blocks from the subnet | `list(string)` | 
[
  ""
]
| no | +| [is\_lb\_private](#input\_is\_lb\_private) | If TRUE, the load balancer scheme will be "internal" else "internet-facing" | `bool` | `null` | no | +| [private\_ssh\_port](#input\_private\_ssh\_port) | Set the SSH port to use between the runner and private instance | `number` | `22` | no | +| [project](#input\_project) | Name of the project | `string` | `"surepay"` | no | +| [public\_ssh\_port](#input\_public\_ssh\_port) | Set the SSH port to use from desktop to the runner | `number` | `22` | no | +| [region](#input\_region) | n/a | `string` | n/a | yes | +| [runner\_additional\_security\_groups](#input\_runner\_additional\_security\_groups) | List of additional security groups to attach to the launch template | `list(string)` | `[]` | no | +| [runner\_ami](#input\_runner\_ami) | The AMI that the runner Host will use. | `string` | `""` | no | +| [runner\_host\_key\_pair](#input\_runner\_host\_key\_pair) | Select the key pair to use to launch the runner host | `string` | n/a | yes | +| [runner\_iam\_policy\_name](#input\_runner\_iam\_policy\_name) | IAM policy name to create for granting the instance role access to the bucket | `string` | `"runnerHost"` | no | +| [runner\_iam\_role\_name](#input\_runner\_iam\_role\_name) | IAM role name to create | `string` | `"runnerRole"` | no | +| [runner\_instance\_count](#input\_runner\_instance\_count) | n/a | `number` | `1` | no | +| [runner\_instance\_count\_max](#input\_runner\_instance\_count\_max) | n/a | `number` | `3` | no | +| [runner\_instance\_count\_min](#input\_runner\_instance\_count\_min) | n/a | `number` | `1` | no | +| [runner\_launch\_template\_name](#input\_runner\_launch\_template\_name) | runner Launch template Name, will also be used for the ASG | `string` | `"runner-lt"` | no | +| [runner\_record\_name](#input\_runner\_record\_name) | DNS record name to use for the runner | `string` | `""` | no | +| [runner\_security\_group\_id](#input\_runner\_security\_group\_id) | Custom security group to use | `string` | `""` | no | +| [ssm\_parameter\_name](#input\_ssm\_parameter\_name) | SSM parameter name for the GitHub Runner token.
Example: "/github/runner/token". | `string` | `""` | no | +| [tags](#input\_tags) | A mapping of tags to assign | `map(string)` | `{}` | no | +| [use\_imds\_v2](#input\_use\_imds\_v2) | Use (IMDSv2) Instance Metadata Service V2 | `bool` | `false` | no | +| [vpc\_id](#input\_vpc\_id) | VPC ID where we'll deploy the runner | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [elb\_arn](#output\_elb\_arn) | The ARN of the ELB for runner hosts | +| [elb\_ip](#output\_elb\_ip) | The DNS name of the ELB for runner hosts | +| [private\_instances\_security\_group](#output\_private\_instances\_security\_group) | The ID of the security group for private instances | +| [runner\_auto\_scaling\_group\_name](#output\_runner\_auto\_scaling\_group\_name) | The name of the Auto Scaling Group for runner hosts | +| [runner\_elb\_id](#output\_runner\_elb\_id) | The ID of the ELB for runner hosts | +| [runner\_host\_security\_group](#output\_runner\_host\_security\_group) | The ID of the runner host security group | +| [target\_group\_arn](#output\_target\_group\_arn) | The ARN of the target group for the ELB | + \ No newline at end of file diff --git a/modules/runner/main.tf b/modules/runner/main.tf index 9a389fe..5fb0ad0 100644 --- a/modules/runner/main.tf +++ b/modules/runner/main.tf @@ -27,7 +27,7 @@ resource "aws_launch_template" "runner_launch_template" { iam_instance_profile { name = aws_iam_instance_profile.runner_host_profile.name } - + key_name = var.runner_host_key_pair user_data = base64encode(templatefile("${path.module}/user_data.sh", { diff --git a/modules/runner/network.tf b/modules/runner/network.tf index 05e35b7..52fa2e9 100644 --- a/modules/runner/network.tf +++ b/modules/runner/network.tf @@ -18,8 +18,8 @@ resource "aws_lb" "runner_lb" { internal = var.is_lb_private name = "${var.project}-lb" - subnets = var.elb_subnets - + subnets = var.elb_subnets + security_groups = [aws_security_group.lb.id] load_balancer_type = "network" tags = merge(var.tags) @@ -35,7 +35,7 @@ resource "aws_lb_target_group" "runner_lb_target_group" { count = var.create_elb ? 1 : 0 name = "${var.project}-lb-target" - port = var.public_ssh_port + port = 22 protocol = "TCP" vpc_id = var.vpc_id target_type = "instance" @@ -57,43 +57,65 @@ resource "aws_lb_listener" "runner_lb_listener_22" { } load_balancer_arn = aws_lb.runner_lb[0].arn - port = var.public_ssh_port + port = 22 protocol = "TCP" } #SGs -resource "aws_security_group" "runner_host_security_group" { - count = var.runner_security_group_id == "" ? 1 : 0 - description = "Enable SSH access to the runner host from external via SSH port" - name = "${var.project}-host" + +#LB +resource "aws_security_group" "lb" { + description = "Enable SSH access to LB via SSH port" + name = "${var.project}-ssh-lb" vpc_id = var.vpc_id tags = merge(var.tags) } +resource "aws_security_group_rule" "shh_lb" { + description = "Incoming traffic to LB" + type = "ingress" + from_port = 22 + to_port = 22 + protocol = "TCP" -resource "aws_security_group" "lb" { - description = "Enable SSH access to LB via SSH port" - name = "${var.project}-ssh-lb" + security_group_id = aws_security_group.lb.id + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "all_egress_lb" { + description = "Outgoing traffic from lb to runner" + type = "egress" + from_port = "0" + to_port = "65535" + protocol = "-1" + # cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.lb.id + source_security_group_id = aws_security_group.runner_host_security_group.id +} + +#runner +resource "aws_security_group" "runner_host_security_group" { + description = "Enable SSH access to the runner host from lb" + name = "${var.project}-runner" vpc_id = var.vpc_id tags = merge(var.tags) } -resource "aws_security_group_rule" "ingress_runner" { - count = var.runner_security_group_id == "" && var.create_elb ? 1 : 0 + +resource "aws_security_group_rule" "lb_runner_ssh" { description = "Incoming traffic to runner" type = "ingress" - from_port = var.public_ssh_port - to_port = var.public_ssh_port + from_port = 22 + to_port = 22 protocol = "TCP" - cidr_blocks = var.ipv4_cidr_block - security_group_id = local.security_group + security_group_id = aws_security_group.runner_host_security_group.id + source_security_group_id = aws_security_group.lb.id } resource "aws_security_group_rule" "egress_runner" { - count = var.runner_security_group_id == "" ? 1 : 0 description = "Outgoing traffic from runner to instances" type = "egress" from_port = "0" @@ -101,9 +123,11 @@ resource "aws_security_group_rule" "egress_runner" { protocol = "-1" cidr_blocks = ["0.0.0.0/0"] - security_group_id = local.security_group + security_group_id = aws_security_group.runner_host_security_group.id + # source_security_group_id = aws_security_group.runner_host_security_group.id } +#priv resource "aws_security_group" "private_instances_security_group" { description = "Enable SSH access to the Private instances from the runner via SSH port" name = "${var.project}-priv-instances" @@ -115,11 +139,10 @@ resource "aws_security_group" "private_instances_security_group" { resource "aws_security_group_rule" "ingress_instances" { description = "Incoming traffic from runner" type = "ingress" - from_port = var.private_ssh_port - to_port = var.private_ssh_port + from_port = 22 + to_port = 22 protocol = "TCP" - source_security_group_id = local.security_group - - security_group_id = aws_security_group.private_instances_security_group.id + security_group_id = aws_security_group.private_instances_security_group.id + source_security_group_id = aws_security_group.runner_host_security_group.id } \ No newline at end of file diff --git a/modules/runner/user_data.sh b/modules/runner/user_data.sh index d6bdc18..7847b7c 100644 --- a/modules/runner/user_data.sh +++ b/modules/runner/user_data.sh @@ -45,7 +45,5 @@ RUNNER_ALLOW_RUNASROOT=true ./config.sh --url ${github_url} --unattended --token # Install as service RUNNER_ALLOW_RUNASROOT=true ./svc.sh install echo ==== ACTIONS-RUNNER DONE ==== -INSTANCE_ID=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-id` -aws ec2 create-tags --region ${aws_region} --resources $INSTANCE_ID --tags Key=terraform-aws-github-runner:setup,Value=done RUNNER_ALLOW_RUNASROOT=true ./run.sh \ No newline at end of file diff --git a/modules/runner/variables.tf b/modules/runner/variables.tf index ec96c5a..30cd71c 100644 --- a/modules/runner/variables.tf +++ b/modules/runner/variables.tf @@ -6,7 +6,7 @@ variable "project" { variable "associate_public_ip_address" { type = bool - default = true + default = false } variable "auto_scaling_group_subnets" { diff --git a/modules/suplement/README.md b/modules/suplement/README.md new file mode 100644 index 0000000..60009f1 --- /dev/null +++ b/modules/suplement/README.md @@ -0,0 +1,41 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_ecr_repository.ecr](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository) | resource | +| [aws_key_pair.keypair](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | +| [aws_ssm_parameter.pat](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [app\_name](#input\_app\_name) | n/a | `any` | n/a | yes | +| [pat\_value](#input\_pat\_value) | n/a | `any` | n/a | yes | +| [project](#input\_project) | n/a | `any` | n/a | yes | +| [public\_key](#input\_public\_key) | n/a | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [ecr\_repository](#output\_ecr\_repository) | The URL of the repository (in the form aws\_account\_id.dkr.ecr.region.amazonaws.com/repositoryName). | +| [key\_pair\_arn](#output\_key\_pair\_arn) | The ARN of the key for the EC2 | +| [key\_pair\_name](#output\_key\_pair\_name) | The name of the key for the EC2 | +| [ssm\_parameter\_name](#output\_ssm\_parameter\_name) | The value of the parameter. | + \ No newline at end of file diff --git a/variables.tf b/variables.tf index 16d7d12..5de2623 100644 --- a/variables.tf +++ b/variables.tf @@ -45,7 +45,7 @@ variable "azs" { #Runner variable "ami_id" { default = "ami-0e9085e60087ce171" #Ubuntu - type = string + type = string } variable "instance_type" { From b7d6eb60e41dadfa7eaf34b22e5a50412b5f9196 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 19:36:12 +0100 Subject: [PATCH 144/148] testing pipe --- main.tf | 1 - modules/runner/README.md | 15 ++++++--------- modules/runner/network.tf | 2 +- modules/runner/variables.tf | 10 +++++----- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/main.tf b/main.tf index 1d14faf..dbd6e9b 100644 --- a/main.tf +++ b/main.tf @@ -42,7 +42,6 @@ module "runner" { vpc_id = module.vpc.vpc_id elb_subnets = module.vpc.public_subnets auto_scaling_group_subnets = module.vpc.private_subnets - ipv4_cidr_block = flatten([module.vpc.private_subnets_cidr_blocks, "0.0.0.0/0"]) github_url = "https://github.com/dilsilva/surepay/settings/actions/runners" github_owner = "dilsilva" diff --git a/modules/runner/README.md b/modules/runner/README.md index b60e366..a24d2f6 100644 --- a/modules/runner/README.md +++ b/modules/runner/README.md @@ -30,11 +30,9 @@ No modules. | [aws_security_group.lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.private_instances_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.runner_host_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group_rule.all_egress_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.egress_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.ingress_instances](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.lb_runner_ssh](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.shh_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.ingress_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_ami.ubuntu-linux-2404](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.assume_policy_document](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -44,7 +42,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | n/a | `bool` | `false` | no | +| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | n/a | `bool` | `true` | no | | [auto\_scaling\_group\_subnets](#input\_auto\_scaling\_group\_subnets) | List of subnets where the Auto Scaling Group will deploy the instances | `list(string)` | n/a | yes | | [aws\_ecs\_cluster\_name](#input\_aws\_ecs\_cluster\_name) | Name of the ECS Cluster where the operations gonna be executed | `string` | `""` | no | | [aws\_ecs\_service\_name](#input\_aws\_ecs\_service\_name) | Name of the ECS service where the operations gonna be executed | `string` | `""` | no | @@ -64,20 +62,19 @@ No modules. | [http\_endpoint](#input\_http\_endpoint) | Whether the metadata service is available | `bool` | `true` | no | | [http\_put\_response\_hop\_limit](#input\_http\_put\_response\_hop\_limit) | The desired HTTP PUT response hop limit for instance metadata requests | `number` | `1` | no | | [instance\_type](#input\_instance\_type) | Instance size of the runner | `string` | `"t3.nano"` | no | -| [ipv4\_cidr\_block](#input\_ipv4\_cidr\_block) | List of ipv4 CIDR blocks from the subnet | `list(string)` | 
[
  ""
]
| no | | [is\_lb\_private](#input\_is\_lb\_private) | If TRUE, the load balancer scheme will be "internal" else "internet-facing" | `bool` | `null` | no | | [private\_ssh\_port](#input\_private\_ssh\_port) | Set the SSH port to use between the runner and private instance | `number` | `22` | no | | [project](#input\_project) | Name of the project | `string` | `"surepay"` | no | | [public\_ssh\_port](#input\_public\_ssh\_port) | Set the SSH port to use from desktop to the runner | `number` | `22` | no | -| [region](#input\_region) | n/a | `string` | n/a | yes | +| [region](#input\_region) | Region of the project | `string` | n/a | yes | | [runner\_additional\_security\_groups](#input\_runner\_additional\_security\_groups) | List of additional security groups to attach to the launch template | `list(string)` | `[]` | no | | [runner\_ami](#input\_runner\_ami) | The AMI that the runner Host will use. | `string` | `""` | no | | [runner\_host\_key\_pair](#input\_runner\_host\_key\_pair) | Select the key pair to use to launch the runner host | `string` | n/a | yes | | [runner\_iam\_policy\_name](#input\_runner\_iam\_policy\_name) | IAM policy name to create for granting the instance role access to the bucket | `string` | `"runnerHost"` | no | | [runner\_iam\_role\_name](#input\_runner\_iam\_role\_name) | IAM role name to create | `string` | `"runnerRole"` | no | -| [runner\_instance\_count](#input\_runner\_instance\_count) | n/a | `number` | `1` | no | -| [runner\_instance\_count\_max](#input\_runner\_instance\_count\_max) | n/a | `number` | `3` | no | -| [runner\_instance\_count\_min](#input\_runner\_instance\_count\_min) | n/a | `number` | `1` | no | +| [runner\_instance\_count](#input\_runner\_instance\_count) | Expected number of instances running on the cluster | `number` | `1` | no | +| [runner\_instance\_count\_max](#input\_runner\_instance\_count\_max) | Maximum expected number of instances running on the cluster | `number` | `3` | no | +| [runner\_instance\_count\_min](#input\_runner\_instance\_count\_min) | Minimun expected number of instances running on the cluster | `number` | `1` | no | | [runner\_launch\_template\_name](#input\_runner\_launch\_template\_name) | runner Launch template Name, will also be used for the ASG | `string` | `"runner-lt"` | no | | [runner\_record\_name](#input\_runner\_record\_name) | DNS record name to use for the runner | `string` | `""` | no | | [runner\_security\_group\_id](#input\_runner\_security\_group\_id) | Custom security group to use | `string` | `""` | no | diff --git a/modules/runner/network.tf b/modules/runner/network.tf index 52fa2e9..d7e7de3 100644 --- a/modules/runner/network.tf +++ b/modules/runner/network.tf @@ -89,7 +89,7 @@ resource "aws_security_group_rule" "all_egress_lb" { from_port = "0" to_port = "65535" protocol = "-1" - # cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.lb.id source_security_group_id = aws_security_group.runner_host_security_group.id } diff --git a/modules/runner/variables.tf b/modules/runner/variables.tf index 30cd71c..8fe4aa7 100644 --- a/modules/runner/variables.tf +++ b/modules/runner/variables.tf @@ -185,11 +185,11 @@ variable "vpc_id" { description = "VPC ID where we'll deploy the runner" } -variable "ipv4_cidr_block" { - type = list(string) - default = [""] - description = "List of ipv4 CIDR blocks from the subnet" -} +# variable "ipv4_cidr_block" { +# type = list(string) +# default = [""] +# description = "List of ipv4 CIDR blocks from the subnet" +# } variable "github_url" { type = string From 6a1fa9df5aa1d3efa3ac3c5fe026b5ac243575fb Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 19:43:02 +0100 Subject: [PATCH 145/148] testing pipeline --- .github/workflows/runner.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml index b82bcd4..7a2a694 100644 --- a/.github/workflows/runner.yaml +++ b/.github/workflows/runner.yaml @@ -1,6 +1,8 @@ name: Deploy to ECR on: push +env: + AWS_REGION: eu-west-1 jobs: build: @@ -36,7 +38,6 @@ jobs: id: login-ecr env: AWS_ACCOUNT: 061051224299 - AWS_REGION: eu-west-1 ECR_REGISTRY: 061051224299.dkr.ecr.eu-west-1.amazonaws.com run: | docker login -u AWS -p $(aws ecr get-login-password --region $AWS_REGION) $ECR_REGISTRY From f5e81738659ad62614d259d25992ae3265a73cb4 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 19:44:30 +0100 Subject: [PATCH 146/148] dev --- modules/runner/variables.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/modules/runner/variables.tf b/modules/runner/variables.tf index 8fe4aa7..2585221 100644 --- a/modules/runner/variables.tf +++ b/modules/runner/variables.tf @@ -185,12 +185,6 @@ variable "vpc_id" { description = "VPC ID where we'll deploy the runner" } -# variable "ipv4_cidr_block" { -# type = list(string) -# default = [""] -# description = "List of ipv4 CIDR blocks from the subnet" -# } - variable "github_url" { type = string default = "" From 2e13a3921667708ba1b6e48bf33d1536fe153732 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 20:18:25 +0100 Subject: [PATCH 147/148] testing pipe --- main.tf | 2 +- modules/ecs/network.tf | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index dbd6e9b..6e3a11d 100644 --- a/main.tf +++ b/main.tf @@ -72,7 +72,7 @@ module "ecs" { private_instances_security_group = module.runner.private_instances_security_group - depends_on = [module.vpc] + depends_on = [module.vpc, module.suplement] } module "suplement" { diff --git a/modules/ecs/network.tf b/modules/ecs/network.tf index a102e5d..d9e5d56 100644 --- a/modules/ecs/network.tf +++ b/modules/ecs/network.tf @@ -1,5 +1,5 @@ resource "aws_security_group" "ecs_lb" { - name = "${var.project}-ecs-lb-sg" + name = "${var.project}-ecs-lb" vpc_id = var.vpc_id ingress { @@ -29,7 +29,7 @@ resource "aws_security_group" "ecs_lb" { } resource "aws_security_group" "ecs" { - name = "${var.project}-ecs-security-group" + name = "${var.project}-ecs" vpc_id = var.vpc_id } @@ -88,6 +88,7 @@ resource "aws_lb" "ecs_alb" { subnets = [for subnet in var.public_subnets : subnet] security_groups = [aws_security_group.ecs.id] + depends_on = [ aws_security_group.ecs_lb ] tags = { Name = "ecs-alb" } @@ -98,6 +99,8 @@ resource "aws_lb_listener" "ecs_alb_listener" { port = 80 protocol = "HTTP" + depends_on = [ aws_security_group.ecs_lb ] + default_action { type = "forward" target_group_arn = aws_lb_target_group.ecs_tg.arn @@ -111,6 +114,8 @@ resource "aws_lb_target_group" "ecs_tg" { target_type = "ip" vpc_id = var.vpc_id + depends_on = [ aws_security_group.ecs_lb ] + health_check { path = "/" } From 4846d597524ee0c591b7615bb06ab6c74e6362c3 Mon Sep 17 00:00:00 2001 From: Diego Silva Date: Sun, 8 Dec 2024 23:25:32 +0100 Subject: [PATCH 148/148] testing pipeline --- modules/ecs/main.tf | 4 ++-- modules/ecs/network.tf | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/ecs/main.tf b/modules/ecs/main.tf index 053e1bc..90f6e38 100644 --- a/modules/ecs/main.tf +++ b/modules/ecs/main.tf @@ -53,8 +53,8 @@ resource "aws_ecs_task_definition" "task" { family = "${var.project}-task" requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" - cpu = 256 - memory = 512 + cpu = 512 + memory = 1024 execution_role_arn = aws_iam_role.task-execution-role.arn container_definitions = jsonencode([ diff --git a/modules/ecs/network.tf b/modules/ecs/network.tf index d9e5d56..797b4ce 100644 --- a/modules/ecs/network.tf +++ b/modules/ecs/network.tf @@ -1,4 +1,5 @@ resource "aws_security_group" "ecs_lb" { + description = "Security group of LB for ECS" name = "${var.project}-ecs-lb" vpc_id = var.vpc_id @@ -29,13 +30,14 @@ resource "aws_security_group" "ecs_lb" { } resource "aws_security_group" "ecs" { + description = "Security group for ECS workloads" name = "${var.project}-ecs" vpc_id = var.vpc_id } resource "aws_security_group_rule" "lb_runner_tcp" { - description = "Incoming shh traffic to ecs" + description = "Incoming shh traffic from LB to ECS" type = "ingress" from_port = 22 to_port = 22 @@ -46,7 +48,7 @@ resource "aws_security_group_rule" "lb_runner_tcp" { } resource "aws_security_group_rule" "lb_runner_ssh" { - description = "Incoming tcp traffic to ecs" + description = "Incoming SSH traffic from LB to ecs" type = "ingress" from_port = 80 to_port = 80 @@ -57,7 +59,7 @@ resource "aws_security_group_rule" "lb_runner_ssh" { } resource "aws_security_group_rule" "lb_runner_ssl" { - description = "Incoming ssl traffic to ecs" + description = "Incoming ssl traffic from LB to ecs" type = "ingress" from_port = 443 to_port = 443 @@ -68,7 +70,7 @@ resource "aws_security_group_rule" "lb_runner_ssl" { } resource "aws_security_group_rule" "egress_runner" { - description = "Outgoing traffic from runner to instances" + description = "Outbound traffic from runner to instances" type = "egress" from_port = "0" to_port = "65535" @@ -86,7 +88,7 @@ resource "aws_lb" "ecs_alb" { internal = false load_balancer_type = "application" subnets = [for subnet in var.public_subnets : subnet] - security_groups = [aws_security_group.ecs.id] + security_groups = [aws_security_group.ecs_lb.id] depends_on = [ aws_security_group.ecs_lb ] tags = {