From 834071822647c85a198c9110638935d83da5ba6b Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 26 Oct 2022 18:24:08 +0200
Subject: [PATCH 01/78] Use preinstalled dependencies for ubuntu

---
 .ci/.e2e-platforms.yaml            |  7 ++++
 .ci/.e2e-tests-tmp.yaml            |  7 ++++
 .ci/Jenkinsfile                    |  9 ++---
 .ci/ansible/github-ssh-keys        |  1 +
 .ci/ansible/playbook.yml           | 53 ++++++++----------------------
 .ci/ansible/tasks/install_deps.yml | 14 ++++----
 .ci/ansible/tasks/runners.yml      | 13 ++++++--
 .ci/packer/aws-ubuntu.pkr.hcl      | 38 +++++++++++++++++++++
 .gitignore                         |  1 +
 9 files changed, 90 insertions(+), 53 deletions(-)
 create mode 100644 .ci/.e2e-tests-tmp.yaml
 create mode 100644 .ci/packer/aws-ubuntu.pkr.hcl

diff --git a/.ci/.e2e-platforms.yaml b/.ci/.e2e-platforms.yaml
index 654f7ed6c3..5d442eed65 100644
--- a/.ci/.e2e-platforms.yaml
+++ b/.ci/.e2e-platforms.yaml
@@ -66,3 +66,10 @@ PLATFORMS:
     instance_type: "t3.xlarge"
     shell_type: sh
     username: ubuntu
+  ubuntu_22_04_amd64_preinstalled:
+    description: "Ubuntu 22.04 AMD64 Preinstalled"
+    image: "ubuntu-2204-e2e-runner-1"
+    instance_type: "t3.xlarge"
+    shell_type: sh
+    username: ubuntu
+    dependencies_installed: true
diff --git a/.ci/.e2e-tests-tmp.yaml b/.ci/.e2e-tests-tmp.yaml
new file mode 100644
index 0000000000..dc388ea694
--- /dev/null
+++ b/.ci/.e2e-tests-tmp.yaml
@@ -0,0 +1,7 @@
+---
+SUITES:  
+  - suite: "fleet"
+    scenarios:
+      - name: "Fleet"
+        tags: "fleet_mode"
+        platforms: ["ubuntu_22_04_amd64_preinstalled"]  
\ No newline at end of file
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index e2da8bda4a..65c9896891 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -506,10 +506,11 @@ def generateFunctionalTestStep(Map args = [:]){
                 log(level: 'DEBUG', text: "Sleeping 300 seconds on Windows so that SSH is accessible in the remote instance.")
                 sleep(300)
               }
-
-              ciBuild() {
-                retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                  sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
+              if (!machine.dependencies_installed) {
+                ciBuild() {
+                  retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                    sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
+                  }
                 }
               }
               ciBuild() {
diff --git a/.ci/ansible/github-ssh-keys b/.ci/ansible/github-ssh-keys
index 382753fe77..d9db4afef9 100644
--- a/.ci/ansible/github-ssh-keys
+++ b/.ci/ansible/github-ssh-keys
@@ -4,4 +4,5 @@ ChrsMark
 juliaElastic
 mdelapenya
 narph
+pazone
 ph
diff --git a/.ci/ansible/playbook.yml b/.ci/ansible/playbook.yml
index f392784d02..538eb7e72f 100644
--- a/.ci/ansible/playbook.yml
+++ b/.ci/ansible/playbook.yml
@@ -2,7 +2,6 @@
   hosts: localhost
   gather_facts: yes
   vars:
-    ansible_user: "{{ nodeUser }}"
   tasks:
   - name: Setup nodes
     include_tasks: tasks/runners.yml
@@ -40,7 +39,6 @@
       vars:
         go_version: 1.17
   vars:
-    ansible_user: "{{ nodeUser }}"
     ansible_python_interpreter: "auto"
   tasks:
   - name: Install deps
@@ -90,6 +88,7 @@
     - vars/main.yml
   tags:
     - setup-node
+    - setup-ami
   pre_tasks:
     - name: Load a variable file based on the OS type
       include_vars: "{{ lookup('first_found', params) }}"
@@ -111,6 +110,7 @@
       include_tasks: tasks/install_deps.yml
       tags:
         - setup-node
+        - setup-ami
     - name: Upgrade pip (CentOS)
       become: True
       ansible.builtin.shell: python3 -m pip install --upgrade pip
@@ -124,16 +124,7 @@
       docker_users:
         - "{{ ansible_user }}"
       when:
-        - '"arm64" not in nodeLabel'
         - 'ansible_os_family not in ["Suse", "Windows"]'
-    - role: geerlingguy.docker_arm
-      become: True
-      docker_daemon_options:
-        default-ulimit: ["nofile=1024000:1024000"]
-      docker_users:
-        - "{{ ansible_user }}"
-      when:
-        - '"arm64" in nodeLabel'
     - role: geerlingguy.helm
       when:
         - ansible_facts['os_family'] != "Windows"
@@ -154,7 +145,6 @@
   vars:
     ansible_python_interpreter: "auto"
     ansible_shell_type: "{{ nodeShellType | default('sh') }}"
-    ansible_user: "{{ nodeUser }}"
     pip_package: "python3-pip"
   tasks:
 
@@ -164,21 +154,19 @@
     with_items:
       - andrewrothstein.kubectl
       - andrewrothstein.kind
+    tags:
+      - setup-ami
     when:
       - ansible_facts['os_family'] != "Windows"
-      - suite is defined
-      - suite in ["kubernetes-autodiscover", "helm"]
-
-  - name: Setup source code
-    include_tasks: tasks/copy_test_files.yml
-    tags:
-      - setup-node
-      - copy-source
 
   - name: Add SSH keys to runner instances
-    include_tasks: tasks/install_ssh_keys.yml
+    authorized_key:
+      user: "{{ ansible_user }}"
+      key: "https://github.com/{{ item }}.keys"
+    with_lines: cat ./github-ssh-keys
     tags:
       - setup-node
+      - setup-ami
 
   - name: Create home dir
     ansible.builtin.file:
@@ -190,6 +178,7 @@
       - setup-node
       - copy-source
       - scripts
+      - setup-ami
     when: ansible_os_family not in ["Windows"]
 
   - name: Create home dir for root
@@ -203,6 +192,7 @@
       - setup-node
       - copy-source
       - scripts
+      - setup-ami
     when: ansible_os_family not in ["Windows"]
 
   - name: Create home dir on Windows
@@ -213,22 +203,5 @@
       - setup-node
       - copy-source
       - scripts
-    when: ansible_os_family in ["Windows"]
-
-  - name: Configure test script
-    include_tasks: tasks/setup_test_script.yml
-    tags:
-      - setup-node
-      - copy-source
-      - scripts
-    when:
-      - ansible_facts['os_family'] != "Windows"
-
-  - name: Configure test script (Windows)
-    include_tasks: tasks/setup_test_script_windows.yml
-    tags:
-      - setup-node
-      - copy-source
-      - scripts
-    when:
-      - ansible_facts['os_family'] == "Windows"
+      - setup-ami
+    when: ansible_os_family in ["Windows"]
\ No newline at end of file
diff --git a/.ci/ansible/tasks/install_deps.yml b/.ci/ansible/tasks/install_deps.yml
index f12f1c58b7..e727cd9d7e 100644
--- a/.ci/ansible/tasks/install_deps.yml
+++ b/.ci/ansible/tasks/install_deps.yml
@@ -107,13 +107,13 @@
     group: "{{ ansible_user }}"
     mode: '0600'
   when: ansible_distribution in ["CentOS", "Debian", "Fedora", "RedHat", "Ubuntu"]
-
-- name: Install Docker for ARM (Debian, Ubuntu)
-  become: True
-  ansible.builtin.shell: curl -fsSL test.docker.com -o get-docker.sh && sh get-docker.sh
-  when:
-    - ansible_distribution in ["Debian", "Ubuntu"]
-    - '"arm64" in nodeLabel'
+# # ARM Docker is installed by geerlingguy.docker
+# - name: Install Docker for ARM (Debian, Ubuntu)
+#   become: True
+#   ansible.builtin.shell: curl -fsSL test.docker.com -o get-docker.sh && sh get-docker.sh
+#   when:
+#     - ansible_distribution in ["Debian", "Ubuntu"]
+#     - '"arm64" in nodeLabel'
 
 - name: Install dependencies on Windows
   chocolatey.chocolatey.win_chocolatey:
diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 2f941ff2f2..2315fc1dde 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -22,13 +22,22 @@
     - provision-stack
     - provision-node
 
-- name: "Create {{nodeLabel}} AWS instances"
+- name: transform AMI name to id
+  amazon.aws.ec2_ami_info:
+    region: "us-east-2"
+    filters:
+      name: '{{nodeImage}}'
+  register: preinstalled_image
+  tags:    
+    - provision-node
+
+- name: "Create preinstalled {{nodeLabel}} AWS instances"
   ec2:
     wait: yes
     key_name: "e2essh-{{runId}}"
     region: us-east-2
     group: e2e
-    image: '{{nodeImage}}'
+    image: "{{ preinstalled_image[0].image_id }}"
     instance_type: '{{nodeInstanceType}}'
     instance_tags:
       branch: "{{branch | default('Not running on CI')}}"
diff --git a/.ci/packer/aws-ubuntu.pkr.hcl b/.ci/packer/aws-ubuntu.pkr.hcl
new file mode 100644
index 0000000000..128a4f0682
--- /dev/null
+++ b/.ci/packer/aws-ubuntu.pkr.hcl
@@ -0,0 +1,38 @@
+packer {
+  required_plugins {
+    amazon = {
+      version = ">= 1.1.5"
+      source  = "github.com/hashicorp/amazon"
+    }
+  }
+}
+
+source "amazon-ebs" "ubuntu" {
+  ami_name      = "ubuntu-2204-e2e-runner-1"
+  instance_type = "t3.xlarge"
+  region        = "us-east-2"
+  source_ami    = "ami-0aeb7c931a5a61206"
+  ssh_username  = "ubuntu"
+  communicator  = "ssh"
+  tags = {
+    OS_Version = "Ubuntu"
+    Release    = "22.04"
+    Arch       = "AMD64"
+  }
+  skip_create_ami = false
+}
+
+build {
+  name = "e2e ubuntu 22.04 AMD64"
+  sources = [
+    "source.amazon-ebs.ubuntu"
+  ]
+
+  provisioner "ansible" {
+    user = "ubuntu"
+    ansible_env_vars  =  ["PACKER_BUILD_NAME={{ build_name }}"]
+    playbook_file     = ".ci/ansible/playbook.yml"
+    extra_arguments   = ["--tags", "setup-ami"]
+    galaxy_file       = ".ci/ansible/requirements.yml"
+  }
+}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index eb45374653..a999f0f8db 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ hosts
 *-sshhosts
 .idea
 .obs
+*.iml

From 86808c7ed26d6ba32bc868fe42c31a9e472920f6 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 27 Oct 2022 16:14:01 +0200
Subject: [PATCH 02/78] Use the new ubuntu image for the stack

---
 .ci/Makefile                  | 6 +++---
 .ci/ansible/tasks/runners.yml | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index dbc8dab94a..5a28fa91df 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -14,10 +14,10 @@ SSH_OPTS=-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
 SSH_OPTS_EXTENDED=-o TCPKeepAlive=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=200 $(SSH_OPTS)
 
 # Debian 10 AMD (see .e2e-platforms.yaml)
-STACK_IMAGE=ami-0d90bed76900e679a
+STACK_IMAGE=ubuntu-2204-e2e-runner-1
 STACK_INSTANCE_TYPE=t3.xlarge
-STACK_LABEL=debian_amd64
-STACK_USER=admin
+STACK_LABEL=ubuntu_2204_preinstalled
+STACK_USER=ubuntu
 STACK_IP_ADDRESS ?= $(shell cat $(PROJECT_DIR)/.ci/.stack-host-ip)
 STACK_INSTANCE_ID ?= stack_$(STACK_LABEL)_$(RUN_ID)
 
diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 2315fc1dde..5d207dea23 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -30,6 +30,7 @@
   register: preinstalled_image
   tags:    
     - provision-node
+    - provision-stack
 
 - name: "Create preinstalled {{nodeLabel}} AWS instances"
   ec2:

From 3c358dc3c37d379a8b35b22c4f0b2e90c36ee561 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 27 Oct 2022 16:31:45 +0200
Subject: [PATCH 03/78] Use latest preinstalled AMI

---
 .ci/ansible/tasks/runners.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 5d207dea23..93b66f6b38 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -27,18 +27,21 @@
     region: "us-east-2"
     filters:
       name: '{{nodeImage}}'
-  register: preinstalled_image
+  register: preinstalled_images
   tags:    
     - provision-node
     - provision-stack
 
 - name: "Create preinstalled {{nodeLabel}} AWS instances"
+  vars:
+    ami_image: >
+      {{ preinstalled_images.images | selectattr('name', 'defined') | sort(attribute='creation_date') | last }}
   ec2:
     wait: yes
     key_name: "e2essh-{{runId}}"
     region: us-east-2
     group: e2e
-    image: "{{ preinstalled_image[0].image_id }}"
+    image: "{{ ami_image }}"
     instance_type: '{{nodeInstanceType}}'
     instance_tags:
       branch: "{{branch | default('Not running on CI')}}"

From 93303fcaa9045a1df402080ffe535d066de8f419 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 27 Oct 2022 17:10:32 +0200
Subject: [PATCH 04/78] Added image_id param

---
 .ci/ansible/tasks/runners.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 93b66f6b38..655ec9328d 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -41,7 +41,7 @@
     key_name: "e2essh-{{runId}}"
     region: us-east-2
     group: e2e
-    image: "{{ ami_image }}"
+    image: "{{ ami_image.image_id }}"
     instance_type: '{{nodeInstanceType}}'
     instance_tags:
       branch: "{{branch | default('Not running on CI')}}"

From 5be1b636982e7e55eeb824bbe5ed578ffab2de97 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 28 Oct 2022 13:32:44 +0200
Subject: [PATCH 05/78] Pick stack image by name

---
 .ci/.e2e-tests-tmp.yaml | 2 +-
 .ci/Jenkinsfile         | 2 +-
 .ci/Makefile            | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.ci/.e2e-tests-tmp.yaml b/.ci/.e2e-tests-tmp.yaml
index dc388ea694..6dab0a1b4f 100644
--- a/.ci/.e2e-tests-tmp.yaml
+++ b/.ci/.e2e-tests-tmp.yaml
@@ -4,4 +4,4 @@ SUITES:
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["ubuntu_22_04_amd64_preinstalled"]  
\ No newline at end of file
+        platforms: ["ubuntu_22_04_amd64_preinstalled"]
\ No newline at end of file
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 65c9896891..0287b64478 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -50,7 +50,7 @@ pipeline {
     booleanParam(name: "SKIP_SCENARIOS", defaultValue: true, description: "If it's needed to skip those scenarios marked as @skip. Default true")
     booleanParam(name: "NIGHTLY_SCENARIOS", defaultValue: false, description: "Deprecated. Not used in this pipeline any more. Please refer to the 'testMatrixFile' param, which defines what scenarios to run")
     string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)')
-    string(name: 'testMatrixFile', defaultValue: '.ci/.e2e-tests.yaml', description: 'The file with the test suite and scenarios to be tested.')
+    string(name: 'testMatrixFile', defaultValue: '.ci/.e2e-tests-tmp.yaml', description: 'The file with the test suite and scenarios to be tested.')
     booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
     booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify to Slack with green builds.")
     string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
diff --git a/.ci/Makefile b/.ci/Makefile
index 5a28fa91df..7cfd7764ad 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -14,10 +14,10 @@ SSH_OPTS=-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
 SSH_OPTS_EXTENDED=-o TCPKeepAlive=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=200 $(SSH_OPTS)
 
 # Debian 10 AMD (see .e2e-platforms.yaml)
-STACK_IMAGE=ubuntu-2204-e2e-runner-1
+STACK_IMAGE=debian-10-amd64-20211011-792
 STACK_INSTANCE_TYPE=t3.xlarge
-STACK_LABEL=ubuntu_2204_preinstalled
-STACK_USER=ubuntu
+STACK_LABEL=debian_amd64
+STACK_USER=admin
 STACK_IP_ADDRESS ?= $(shell cat $(PROJECT_DIR)/.ci/.stack-host-ip)
 STACK_INSTANCE_ID ?= stack_$(STACK_LABEL)_$(RUN_ID)
 

From 1f81e3e86bd3495d5b239ec614435a0d5510a298 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 28 Oct 2022 16:19:25 +0200
Subject: [PATCH 06/78] Ansible: removed intermediate nodeUser variable in
 Ansible to use ansible_user directly

---
 .ci/Makefile                       | 40 +++++++++++++++---------------
 .ci/ansible/elastic-stack.yml      |  2 +-
 .ci/ansible/fetch-test-reports.yml |  2 +-
 .ci/ansible/run-tests.yml          |  2 +-
 .ci/ansible/tasks/runners.yml      |  2 +-
 .ci/ansible/teardown.yml           |  4 +--
 6 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 7cfd7764ad..5f93ba2e94 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -85,9 +85,9 @@ set-env-%:
 .PHONY: provision-stack
 provision-stack: .runID
 	@:$(call check_defined, RUN_ID, You need to an unique RUN_ID. To create it please run 'make .runID' goal)
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml --user $(STACK_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
+		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE)" \
 		--extra-vars "runId=$(RUN_ID) instanceID=$(STACK_INSTANCE_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t provision-stack
@@ -97,9 +97,9 @@ setup-stack: export TAGS = non-existing-tag
 setup-stack: .runID
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml --user $(STACK_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
+		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE)"  \
 		--extra-vars "runId=$(RUN_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t setup-stack \
@@ -111,9 +111,9 @@ create-stack: provision-stack setup-stack start-elastic-stack
 .PHONY: destroy-stack
 destroy-stack:
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml --user $(STACK_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars="$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeUser=$(STACK_IMAGE)" \
+		--extra-vars="$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE)" \
 		--extra-vars="runId=$(RUN_ID) instanceID=$(STACK_INSTANCE_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t destroy
@@ -135,9 +135,9 @@ show-stack:
 provision-node: .runID set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
-	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
+	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml --user $(NODE_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
+		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
 		--extra-vars "runId=$(RUN_ID) instanceID=$(NODE_INSTANCE_ID) nodeShellType=$${NODE_SHELL_TYPE} workspace=$(PROJECT_DIR)/ suite=$(SUITE) sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t provision-node
@@ -148,9 +148,9 @@ setup-node: .runID set-env-$(NODE_LABEL)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
 	@:$(call check_defined, NODE_IP_ADDRESS, IP address of the test node not defined)
 	source $(VENV)/bin/activate; \
-	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
+	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml --user $(NODE_USER)\
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
+		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
 		--extra-vars "runId=$(RUN_ID) nodeShellType=$${NODE_SHELL_TYPE} workspace=$(PROJECT_DIR)/ suite=$(SUITE) sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='-o ConnectTimeout=180s $(SSH_OPTS)' \
 		-t setup-node \
@@ -162,9 +162,9 @@ create-node: provision-node setup-node
 .PHONY: destroy-node
 destroy-node: set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
-	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/teardown.yml \
+	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/teardown.yml --user $(NODE_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars="$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeUser=$${NODE_USER}" \
+		--extra-vars="$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE}" \
 		--extra-vars="runId=$(RUN_ID) instanceID=$(NODE_INSTANCE_ID) sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)'
 	rm -fr $(PROJECT_DIR)/.ci/.node-host-ip
@@ -173,9 +173,9 @@ destroy-node: set-env-$(NODE_LABEL)
 fetch-test-reports: .runID set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, NODE_IP_ADDRESS, IP address of the test node not defined)
-	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/fetch-test-reports.yml \
+	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/fetch-test-reports.yml --user $(NODE_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
+		--extra-vars "$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
 		--extra-vars "runId=$(RUN_ID) nodeShellType=$${NODE_SHELL_TYPE} workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t fetch-reports \
@@ -202,9 +202,9 @@ destroy-elastic-stack:
 start-elastic-stack:
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	PROVIDER="remote" SUITE="$(SUITE)" TAGS="non-existent-tag" \
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/elastic-stack.yml \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/elastic-stack.yml --user $(STACK_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
+		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE)" \
 		--extra-vars "runId=$(RUN_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t elastic-stack \
@@ -220,9 +220,9 @@ recreate-fleet-server:
 run-tests: set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && PROVIDER="$(PROVIDER)" SUITE="$(SUITE)" TAGS="$(TAGS)" REPORT_PREFIX="$(SUITE)_$${NODE_LABEL}_$(TAGS)" \
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml --user $(NODE_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
+		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
 		--extra-vars "runId=$(RUN_ID) nodeShellType=$${NODE_SHELL_TYPE} workspace=$(PROJECT_DIR)/ suite=$(SUITE) sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t run-tests \
@@ -233,9 +233,9 @@ run-tests: set-env-$(NODE_LABEL)
 run-tests-win: set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && PROVIDER="$(PROVIDER)" SUITE="$(SUITE)" TAGS="$(TAGS)" REPORT_PREFIX="$(SUITE)_$${NODE_LABEL}_$(TAGS)" \
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml --user $(NODE_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
+		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
 		--extra-vars "runId=$(RUN_ID) nodeShellType=$${NODE_SHELL_TYPE} workspace=$(PROJECT_DIR)/ suite=$(SUITE) sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t run-tests \
diff --git a/.ci/ansible/elastic-stack.yml b/.ci/ansible/elastic-stack.yml
index bc738bd68c..cdddb61617 100644
--- a/.ci/ansible/elastic-stack.yml
+++ b/.ci/ansible/elastic-stack.yml
@@ -21,7 +21,7 @@
   vars:
     ansible_python_interpreter: "auto"
     ansible_shell_type: "{{ nodeShellType | default('sh') }}"
-    ansible_user: "{{ nodeUser }}"
+#    ansible_user: "{{ nodeUser }}"
     pip_package: "python3-pip"
   tasks:
 
diff --git a/.ci/ansible/fetch-test-reports.yml b/.ci/ansible/fetch-test-reports.yml
index 95aa7b634a..e3792c7015 100644
--- a/.ci/ansible/fetch-test-reports.yml
+++ b/.ci/ansible/fetch-test-reports.yml
@@ -21,7 +21,7 @@
   vars:
     ansible_python_interpreter: "auto"
     ansible_shell_type: "{{ nodeShellType | default('sh') }}"
-    ansible_user: "{{ nodeUser }}"
+#    ansible_user: "{{ nodeUser }}"
     pip_package: "python3-pip"
   tasks:
   - name: Fetch test reports
diff --git a/.ci/ansible/run-tests.yml b/.ci/ansible/run-tests.yml
index ecb706e46b..fe6b3a43c9 100644
--- a/.ci/ansible/run-tests.yml
+++ b/.ci/ansible/run-tests.yml
@@ -21,7 +21,7 @@
   vars:
     ansible_python_interpreter: "auto"
     ansible_shell_type: "{{ nodeShellType | default('sh') }}"
-    ansible_user: "{{ nodeUser }}"
+#    ansible_user: "{{ nodeUser }}"
     pip_package: "python3-pip"
   tasks:
   - name: Setup source code
diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 655ec9328d..e09c6b30ea 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -81,7 +81,7 @@
 - name: Add AWS host to ssh address list
   lineinfile:
     state: present
-    line: "- {{ nodeUser }}@{{addr.public_ip}}"
+    line: "- {{ ansible_user }}@{{addr.public_ip}}"
     insertafter: EOF
     dest: "{{ workspace }}{{nodeLabel}}-sshhosts"
     create: yes
diff --git a/.ci/ansible/teardown.yml b/.ci/ansible/teardown.yml
index ac2a829d53..148b731d8a 100644
--- a/.ci/ansible/teardown.yml
+++ b/.ci/ansible/teardown.yml
@@ -1,8 +1,8 @@
 ---
 - name: Teardown environment
   hosts: localhost
-  vars:
-    ansible_user: "{{ nodeUser }}"
+#  vars:
+#    ansible_user: "{{ nodeUser }}"
   tasks:
   - name: Gather information about any instance with a tag key 'name' and value "e2e-{{ instanceID }}"
     amazon.aws.ec2_instance_info:

From 00f294d8c5d1ee73503581f6528c78327c366a7d Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 28 Oct 2022 17:09:08 +0200
Subject: [PATCH 07/78] Ansible: added default value for ansible_user for
 docker role

---
 .ci/ansible/playbook.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.ci/ansible/playbook.yml b/.ci/ansible/playbook.yml
index 538eb7e72f..d7e6f5c896 100644
--- a/.ci/ansible/playbook.yml
+++ b/.ci/ansible/playbook.yml
@@ -28,8 +28,7 @@
       become: True
       docker_daemon_options:
         default-ulimit: ["nofile=1024000:1024000"]
-      docker_users:
-        - "{{ ansible_user }}"
+      docker_users: ["{{ ansible_user | default(ansible_env.USER) }}"]
     - role: geerlingguy.pip
       become: True
     - role: andrewrothstein.kubectl

From 67e4d9a02a020efee4107ec31917286b2061dede Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 28 Oct 2022 17:28:15 +0200
Subject: [PATCH 08/78] Ansible: added default value for ansible_user for
 main.yml

---
 .ci/ansible/vars/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/ansible/vars/main.yml b/.ci/ansible/vars/main.yml
index 9f120b3e94..d70c442d81 100644
--- a/.ci/ansible/vars/main.yml
+++ b/.ci/ansible/vars/main.yml
@@ -1,5 +1,5 @@
 ---
 e2e_project_name: "e2e-testing"
 
-e2e_base_dir: "/home/{{ ansible_user }}/{{ e2e_project_name }}/"
-e2e_home_dir: "/home/{{ ansible_user }}/.op/"
+e2e_base_dir: "/home/{{ ansible_user | default(ansible_env.USER) }}/{{ e2e_project_name }}/"
+e2e_home_dir: "/home/{{ ansible_user | default(ansible_env.USER) }}/.op/"

From 7eff6472e21f6806da07ece30e3f0f7a04f68be2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 31 Oct 2022 14:35:45 +0100
Subject: [PATCH 09/78] Ansible: test change

---
 .ci/ansible/github-ssh-keys | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.ci/ansible/github-ssh-keys b/.ci/ansible/github-ssh-keys
index d9db4afef9..5b0c626f85 100644
--- a/.ci/ansible/github-ssh-keys
+++ b/.ci/ansible/github-ssh-keys
@@ -2,7 +2,6 @@ adam-stokes
 AndersonQ
 ChrsMark
 juliaElastic
-mdelapenya
 narph
 pazone
 ph

From fcef5a4348549113d6fc80164be35cc26373bbc5 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 31 Oct 2022 17:19:32 +0100
Subject: [PATCH 10/78] Ansible: disabled ssh-import-id installation

---
 .ci/ansible/tasks/install_deps.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.ci/ansible/tasks/install_deps.yml b/.ci/ansible/tasks/install_deps.yml
index e727cd9d7e..cffee69664 100644
--- a/.ci/ansible/tasks/install_deps.yml
+++ b/.ci/ansible/tasks/install_deps.yml
@@ -92,11 +92,12 @@
   until: package_install_res is success
   when: ansible_distribution in ["OracleLinux"]
 
-- name: Install ssh-import-id python package to copy public SSH keys from Github accounts
-  become: True
-  pip:
-    name: ssh-import-id
-  when: ansible_distribution not in ["Windows"]
+# Not required since authorized_key is used
+#- name: Install ssh-import-id python package to copy public SSH keys from Github accounts
+#  become: True
+#  pip:
+#    name: ssh-import-id
+#  when: ansible_distribution not in ["Windows"]
 
 - name: Set sshd configuration for client alive settings
   become: True

From ba1c808d33ee7f993e6ea8c782ae9f0d5e348f6a Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 2 Nov 2022 14:21:05 +0100
Subject: [PATCH 11/78] Ansible: disabled ssh-import-id installation

---
 .ci/ansible/playbook.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.ci/ansible/playbook.yml b/.ci/ansible/playbook.yml
index d7e6f5c896..400d142251 100644
--- a/.ci/ansible/playbook.yml
+++ b/.ci/ansible/playbook.yml
@@ -74,8 +74,11 @@
     tags:
       - setup-stack
 
-  - name: Add SSH keys to stack
-    include_tasks: tasks/install_ssh_keys.yml
+  - name: Add SSH keys to runner instances
+    authorized_key:
+      user: "{{ ansible_user }}"
+      key: "https://github.com/{{ item }}.keys"
+    with_lines: cat ./github-ssh-keys
     tags:
       - setup-stack
 

From 4b6ab6c6246fb23b1418a824a5c831deab7a1f53 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 2 Nov 2022 15:07:50 +0100
Subject: [PATCH 12/78] test: execute setup-node on preinstalled runner machine

---
 .ci/Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 0287b64478..f34c0156a9 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -506,13 +506,13 @@ def generateFunctionalTestStep(Map args = [:]){
                 log(level: 'DEBUG', text: "Sleeping 300 seconds on Windows so that SSH is accessible in the remote instance.")
                 sleep(300)
               }
-              if (!machine.dependencies_installed) {
+//               if (!machine.dependencies_installed) {
                 ciBuild() {
                   retryWithSleep(retries: 3, seconds: 5, backoff: true){
                     sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
                   }
                 }
-              }
+//               }
               ciBuild() {
                 sh(label: 'Run tests in the node', script: "make -C .ci ${runCommand}")
               }

From a54f5895a40fbe3d6e27e1967dd544e9e656973b Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 3 Nov 2022 12:04:05 +0100
Subject: [PATCH 13/78] removed sshd killing task

---
 .ci/ansible/tasks/install_deps.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.ci/ansible/tasks/install_deps.yml b/.ci/ansible/tasks/install_deps.yml
index cffee69664..ade5b5af13 100644
--- a/.ci/ansible/tasks/install_deps.yml
+++ b/.ci/ansible/tasks/install_deps.yml
@@ -99,15 +99,15 @@
 #    name: ssh-import-id
 #  when: ansible_distribution not in ["Windows"]
 
-- name: Set sshd configuration for client alive settings
-  become: True
-  ansible.builtin.copy:
-    src: sshd_config
-    dest: /etc/ssh/sshd_config
-    owner: "{{ ansible_user }}"
-    group: "{{ ansible_user }}"
-    mode: '0600'
-  when: ansible_distribution in ["CentOS", "Debian", "Fedora", "RedHat", "Ubuntu"]
+#- name: Set sshd configuration for client alive settings
+#  become: True
+#  ansible.builtin.copy:
+#    src: sshd_config
+#    dest: /etc/ssh/sshd_config
+#    owner: "{{ ansible_user }}"
+#    group: "{{ ansible_user }}"
+#    mode: '0600'
+#  when: ansible_distribution in ["CentOS", "Debian", "Fedora", "RedHat", "Ubuntu"]
 # # ARM Docker is installed by geerlingguy.docker
 # - name: Install Docker for ARM (Debian, Ubuntu)
 #   become: True

From a4de617e4c4f882220dfde41e0cd3b7eaefe2560 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 3 Nov 2022 13:41:46 +0100
Subject: [PATCH 14/78] Skip setup-node when dependencies are preinstalled

---
 .ci/Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index f34c0156a9..0287b64478 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -506,13 +506,13 @@ def generateFunctionalTestStep(Map args = [:]){
                 log(level: 'DEBUG', text: "Sleeping 300 seconds on Windows so that SSH is accessible in the remote instance.")
                 sleep(300)
               }
-//               if (!machine.dependencies_installed) {
+              if (!machine.dependencies_installed) {
                 ciBuild() {
                   retryWithSleep(retries: 3, seconds: 5, backoff: true){
                     sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
                   }
                 }
-//               }
+              }
               ciBuild() {
                 sh(label: 'Run tests in the node', script: "make -C .ci ${runCommand}")
               }

From f2a507d5d04a14fc9bc2096b48e175a13f57abdd Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 3 Nov 2022 17:03:33 +0100
Subject: [PATCH 15/78] Run all tests on ubuntu preinstalled

---
 .ci/.e2e-tests-tmp.yaml | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/.ci/.e2e-tests-tmp.yaml b/.ci/.e2e-tests-tmp.yaml
index 6dab0a1b4f..08f638d5d4 100644
--- a/.ci/.e2e-tests-tmp.yaml
+++ b/.ci/.e2e-tests-tmp.yaml
@@ -1,7 +1,37 @@
 ---
-SUITES:  
+SUITES:
+  - suite: "helm"
+    provider: "docker"
+    scenarios:
+      - name: "APM Server"
+        tags: "apm-server"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+      - name: "Filebeat"
+        tags: "filebeat"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+      - name: "Metricbeat"
+        tags: "metricbeat"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["ubuntu_22_04_amd64_preinstalled"]
\ No newline at end of file
+        platforms: ["ubuntu_22_04_amd64_preinstalled"]
+      - name: "Upgrade Agent"
+        tags: "upgrade_agent"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+  - suite: "kubernetes-autodiscover"
+    provider: "docker"
+    scenarios:
+      - name: "kubernetes autodiscover with elastic-agent"
+        tags: "elastic-agent"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+      - name: "kubernetes autodiscover with filebeat"
+        tags: "filebeat"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+      - name: "kubernetes autodiscover with heartbeat"
+        tags: "heartbeat"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+      - name: "kubernetes autodiscover with metricbeat"
+        tags: "metricbeat"
+        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
\ No newline at end of file

From 1a1f026aef8dc849fe8b7708cff74c3521e91505 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 3 Nov 2022 18:00:48 +0100
Subject: [PATCH 16/78] Make helm always installed

---
 .ci/ansible/playbook.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.ci/ansible/playbook.yml b/.ci/ansible/playbook.yml
index 400d142251..b878783039 100644
--- a/.ci/ansible/playbook.yml
+++ b/.ci/ansible/playbook.yml
@@ -130,8 +130,6 @@
     - role: geerlingguy.helm
       when:
         - ansible_facts['os_family'] != "Windows"
-        - suite is defined
-        - suite in ["kubernetes-autodiscover", "helm"]
     - role: mdelapenya.go
       become: True
       when: ansible_facts['os_family'] != "Windows"

From 0c10008a4724c3b26f3899ace70f3d385b65f1ba Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 3 Nov 2022 18:15:25 +0100
Subject: [PATCH 17/78] Introduced variables

---
 .../{aws-ubuntu.pkr.hcl => aws-runners.pkr.hcl}   | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
 rename .ci/packer/{aws-ubuntu.pkr.hcl => aws-runners.pkr.hcl} (75%)

diff --git a/.ci/packer/aws-ubuntu.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
similarity index 75%
rename from .ci/packer/aws-ubuntu.pkr.hcl
rename to .ci/packer/aws-runners.pkr.hcl
index 128a4f0682..bf8e6d4cf4 100644
--- a/.ci/packer/aws-ubuntu.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -7,10 +7,20 @@ packer {
   }
 }
 
+variable "skip_create_ami" {
+  type = bool
+  default = false
+}
+
+locals {
+  aws_region = "us-east-2"
+  force_deregister = true
+}
+
 source "amazon-ebs" "ubuntu" {
   ami_name      = "ubuntu-2204-e2e-runner-1"
   instance_type = "t3.xlarge"
-  region        = "us-east-2"
+  region        = local.aws_region
   source_ami    = "ami-0aeb7c931a5a61206"
   ssh_username  = "ubuntu"
   communicator  = "ssh"
@@ -19,7 +29,8 @@ source "amazon-ebs" "ubuntu" {
     Release    = "22.04"
     Arch       = "AMD64"
   }
-  skip_create_ami = false
+  skip_create_ami = var.skip_create_ami
+  force_deregister = local.force_deregister
 }
 
 build {

From dbac069db2ef2e83c820d72f89c741ae885ed522 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 7 Nov 2022 13:06:18 +0100
Subject: [PATCH 18/78] Debian 10, 11 and arm

---
 .ci/.e2e-platforms.yaml        | 21 +++++++++++++
 .ci/.e2e-tests-tmp.yaml        | 18 +++++------
 .ci/packer/aws-runners.pkr.hcl | 57 ++++++++++++++++++++++++++++++++--
 3 files changed, 84 insertions(+), 12 deletions(-)

diff --git a/.ci/.e2e-platforms.yaml b/.ci/.e2e-platforms.yaml
index 5d442eed65..56de252bbe 100644
--- a/.ci/.e2e-platforms.yaml
+++ b/.ci/.e2e-platforms.yaml
@@ -73,3 +73,24 @@ PLATFORMS:
     shell_type: sh
     username: ubuntu
     dependencies_installed: true
+  debian_10_arm64_preinstalled:
+    description: "Debian 10 ARM64 Preinstalled"
+    image: "debian-10-arm64-runner-1"
+    instance_type: "a1.large"
+    shell_type: sh
+    username: admin
+    dependencies_installed: true
+  debian_10_amd64_preinstalled:
+    description: "Debian 10 AMD64 Preinstalled"
+    image: "debian-10-amd64-runner-1"
+    instance_type: "t3.xlarge"
+    shell_type: sh
+    username: admin
+    dependencies_installed: true
+  debian_11_amd64_preinstalled:
+    description: "Debian 11 AMD64 Preinstalled"
+    image: "debian-11-amd64-runner-1"
+    instance_type: "t3.xlarge"
+    shell_type: sh
+    username: admin
+    dependencies_installed: true
diff --git a/.ci/.e2e-tests-tmp.yaml b/.ci/.e2e-tests-tmp.yaml
index 08f638d5d4..c63306c3db 100644
--- a/.ci/.e2e-tests-tmp.yaml
+++ b/.ci/.e2e-tests-tmp.yaml
@@ -5,33 +5,33 @@ SUITES:
     scenarios:
       - name: "APM Server"
         tags: "apm-server"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+        platforms: [ "debian_10_amd64_preinstalled" ]
       - name: "Filebeat"
         tags: "filebeat"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+        platforms: [ "debian_10_amd64_preinstalled" ]
       - name: "Metricbeat"
         tags: "metricbeat"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+        platforms: [ "debian_10_amd64_preinstalled" ]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["ubuntu_22_04_amd64_preinstalled"]
+        platforms: ["debian_10_arm64_preinstalled", "debian_11_amd64_preinstalled"]
       - name: "Upgrade Agent"
         tags: "upgrade_agent"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+        platforms: [ "debian_10_amd64_preinstalled" ]
   - suite: "kubernetes-autodiscover"
     provider: "docker"
     scenarios:
       - name: "kubernetes autodiscover with elastic-agent"
         tags: "elastic-agent"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+        platforms: [ "debian_10_amd64_preinstalled" ]
       - name: "kubernetes autodiscover with filebeat"
         tags: "filebeat"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+        platforms: [ "debian_10_amd64_preinstalled" ]
       - name: "kubernetes autodiscover with heartbeat"
         tags: "heartbeat"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
+        platforms: [ "debian_10_amd64_preinstalled" ]
       - name: "kubernetes autodiscover with metricbeat"
         tags: "metricbeat"
-        platforms: [ "ubuntu_22_04_amd64_preinstalled" ]
\ No newline at end of file
+        platforms: [ "debian_10_amd64_preinstalled" ]
\ No newline at end of file
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index bf8e6d4cf4..3d99ba06db 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -33,14 +33,65 @@ source "amazon-ebs" "ubuntu" {
   force_deregister = local.force_deregister
 }
 
+source "amazon-ebs" "debian-10-amd64" {
+  ami_name      = "debian-10-amd64-runner-1"
+  instance_type = "t3.xlarge"
+  region        = local.aws_region
+  source_ami    = "ami-0d90bed76900e679a"
+  ssh_username  = "admin"
+  communicator  = "ssh"
+  tags = {
+    OS_Version = "Debian"
+    Release    = "10"
+    Arch       = "AMD64"
+  }
+  skip_create_ami = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
+source "amazon-ebs" "debian-10-arm64" {
+  ami_name      = "debian-10-arm64-runner-1"
+  instance_type = "a1.large"
+  region        = local.aws_region
+  source_ami    = "ami-06dac44ad759182bd"
+  ssh_username  = "admin"
+  communicator  = "ssh"
+  tags = {
+    OS_Version = "Debian"
+    Release    = "10"
+    Arch       = "AMD64"
+  }
+  skip_create_ami = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
+source "amazon-ebs" "debian-11-amd64" {
+  ami_name      = "debian-11-amd64-runner-1"
+  instance_type = "t3.xlarge"
+  region        = local.aws_region
+  source_ami    = "ami-0c7c4e3c6b4941f0f"
+  ssh_username  = "admin"
+  communicator  = "ssh"
+  tags          = {
+    OS_Version = "Debian"
+    Release    = "10"
+    Arch       = "AMD64"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
 build {
-  name = "e2e ubuntu 22.04 AMD64"
+  name = "e2e runners AMIs"
   sources = [
-    "source.amazon-ebs.ubuntu"
+    "source.amazon-ebs.ubuntu",
+    "source.amazon-ebs.debian-10-amd64",
+    "source.amazon-ebs.debian-10-arm64",
+    "source.amazon-ebs.debian-11-amd64"
   ]
 
   provisioner "ansible" {
-    user = "ubuntu"
+    user = build.User
     ansible_env_vars  =  ["PACKER_BUILD_NAME={{ build_name }}"]
     playbook_file     = ".ci/ansible/playbook.yml"
     extra_arguments   = ["--tags", "setup-ami"]

From 922e0d5c6029bd54faf9843c480f6eb1a590a2e3 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 7 Nov 2022 15:08:34 +0100
Subject: [PATCH 19/78] CentOs8 AMD+ARM, Windows2019

---
 .ci/.e2e-platforms.yaml                | 15 ++++++
 .ci/.e2e-tests-tmp.yaml                |  8 +--
 .ci/packer/aws-runners.pkr.hcl         | 67 +++++++++++++++++++++++---
 .ci/packer/scripts/winrm_bootstrap.txt | 47 ++++++++++++++++++
 4 files changed, 127 insertions(+), 10 deletions(-)
 create mode 100644 .ci/packer/scripts/winrm_bootstrap.txt

diff --git a/.ci/.e2e-platforms.yaml b/.ci/.e2e-platforms.yaml
index 56de252bbe..1f505ec853 100644
--- a/.ci/.e2e-platforms.yaml
+++ b/.ci/.e2e-platforms.yaml
@@ -94,3 +94,18 @@ PLATFORMS:
     shell_type: sh
     username: admin
     dependencies_installed: true
+  centos8_arm64_preinstalled:
+    description: "CentOS Stream 8 aarch64 preinstalled"
+    image: "centos-8-arm64-runner-1"
+    instance_type: "a1.large"
+    shell_type: sh
+    username: centos
+    dependencies_installed: true
+  centos8_amd64_preinstalled:
+    description: "CentOS Stream 8 x86_64"
+    image: "centos-8-amd64-runner-1"
+    instance_type: "t3.xlarge"
+    shell_type: sh
+    username: centos
+    dependencies_installed: true
+
diff --git a/.ci/.e2e-tests-tmp.yaml b/.ci/.e2e-tests-tmp.yaml
index c63306c3db..7542ed1822 100644
--- a/.ci/.e2e-tests-tmp.yaml
+++ b/.ci/.e2e-tests-tmp.yaml
@@ -16,10 +16,10 @@ SUITES:
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["debian_10_arm64_preinstalled", "debian_11_amd64_preinstalled"]
-      - name: "Upgrade Agent"
-        tags: "upgrade_agent"
-        platforms: [ "debian_10_amd64_preinstalled" ]
+        platforms: ["centos8_arm64_preinstalled", "centos8_amd64_preinstalled"]
+#      - name: "Upgrade Agent"
+#        tags: "upgrade_agent"
+#        platforms: [ "debian_10_amd64_preinstalled" ]
   - suite: "kubernetes-autodiscover"
     provider: "docker"
     scenarios:
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 3d99ba06db..0ce26ab786 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -59,7 +59,7 @@ source "amazon-ebs" "debian-10-arm64" {
   tags = {
     OS_Version = "Debian"
     Release    = "10"
-    Arch       = "AMD64"
+    Arch       = "ARM64"
   }
   skip_create_ami = var.skip_create_ami
   force_deregister = local.force_deregister
@@ -74,20 +74,75 @@ source "amazon-ebs" "debian-11-amd64" {
   communicator  = "ssh"
   tags          = {
     OS_Version = "Debian"
-    Release    = "10"
+    Release    = "11"
+    Arch       = "AMD64"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
+source "amazon-ebs" "centos-8-amd64" {
+  ami_name      = "centos-8-amd64-runner-1"
+  instance_type = "t3.xlarge"
+  region        = local.aws_region
+  source_ami    = "ami-045b0a05944af45c1"
+  ssh_username  = "centos"
+  communicator  = "ssh"
+  tags          = {
+    OS_Version = "Centos"
+    Release    = "8"
     Arch       = "AMD64"
   }
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
+source "amazon-ebs" "centos-8-arm64" {
+  ami_name      = "centos-8-arm64-runner-1"
+  instance_type = "a1.large"
+  region        = local.aws_region
+  source_ami    = "ami-01cdc9e8306344fe0"
+  ssh_username  = "centos"
+  communicator  = "ssh"
+  tags          = {
+    OS_Version = "Centos"
+    Release    = "8"
+    Arch       = "ARM64"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
+source "amazon-ebs" "windows2019" {
+  ami_name        = "windows-2019-runner-1"
+  instance_type   = "c5.2xlarge"
+  region          = local.aws_region
+  source_ami      = "ami-0587bd602f1da2f1d"
+  winrm_username  = "ogc"
+  winrm_insecure  = true
+  winrm_use_ssl   = true
+  communicator    = "winrm"
+  user_data_file  = ".ci/packer/scripts/winrm_bootstrap.txt"
+  tags            = {
+    OS_Version  = "Windows"
+    Release     = "9"
+    Arch        = "AMD64"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
 build {
   name = "e2e runners AMIs"
   sources = [
-    "source.amazon-ebs.ubuntu",
-    "source.amazon-ebs.debian-10-amd64",
-    "source.amazon-ebs.debian-10-arm64",
-    "source.amazon-ebs.debian-11-amd64"
+#    "source.amazon-ebs.ubuntu",
+#    "source.amazon-ebs.debian-10-amd64",
+#    "source.amazon-ebs.debian-10-arm64",
+#    "source.amazon-ebs.debian-11-amd64",
+    "source.amazon-ebs.centos-8-amd64",
+    "source.amazon-ebs.centos-8-arm64"
+#    "source.amazon-ebs.windows2019"
+
   ]
 
   provisioner "ansible" {
diff --git a/.ci/packer/scripts/winrm_bootstrap.txt b/.ci/packer/scripts/winrm_bootstrap.txt
new file mode 100644
index 0000000000..074b42e285
--- /dev/null
+++ b/.ci/packer/scripts/winrm_bootstrap.txt
@@ -0,0 +1,47 @@
+<powershell>
+
+# MAKE SURE IN YOUR PACKER CONFIG TO SET:
+#
+#
+#    "winrm_username": "Administrator",
+#    "winrm_insecure": true,
+#    "winrm_use_ssl": true,
+#
+#
+
+
+write-output "Running User Data Script"
+write-host "(host) Running User Data Script"
+
+Set-ExecutionPolicy Unrestricted -Scope LocalMachine -Force -ErrorAction Ignore
+
+# Don't set this before Set-ExecutionPolicy as it throws an error
+$ErrorActionPreference = "stop"
+
+# Remove HTTP listener
+Remove-Item -Path WSMan:\Localhost\listener\listener* -Recurse
+
+# Create a self-signed certificate to let ssl work
+$Cert = New-SelfSignedCertificate -CertstoreLocation Cert:\LocalMachine\My -DnsName "packer"
+New-Item -Path WSMan:\LocalHost\Listener -Transport HTTPS -Address * -CertificateThumbPrint $Cert.Thumbprint -Force
+
+# WinRM
+write-output "Setting up WinRM"
+write-host "(host) setting up WinRM"
+
+cmd.exe /c winrm quickconfig -q
+cmd.exe /c winrm set "winrm/config" '@{MaxTimeoutms="1800000"}'
+cmd.exe /c winrm set "winrm/config/winrs" '@{MaxMemoryPerShellMB="1024"}'
+cmd.exe /c winrm set "winrm/config/service" '@{AllowUnencrypted="true"}'
+cmd.exe /c winrm set "winrm/config/client" '@{AllowUnencrypted="true"}'
+cmd.exe /c winrm set "winrm/config/service/auth" '@{Basic="true"}'
+cmd.exe /c winrm set "winrm/config/client/auth" '@{Basic="true"}'
+cmd.exe /c winrm set "winrm/config/service/auth" '@{CredSSP="true"}'
+cmd.exe /c winrm set "winrm/config/listener?Address=*+Transport=HTTPS" "@{Port=`"5986`";Hostname=`"packer`";CertificateThumbprint=`"$($Cert.Thumbprint)`"}"
+cmd.exe /c netsh advfirewall firewall set rule group="remote administration" new enable=yes
+cmd.exe /c netsh firewall add portopening TCP 5986 "Port 5986"
+cmd.exe /c net stop winrm
+cmd.exe /c sc config winrm start= auto
+cmd.exe /c net start winrm
+
+</powershell>

From 13a3f2247533164f0e7a89b03b44b8769e508774 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 10 Nov 2022 14:44:13 +0100
Subject: [PATCH 20/78] Added explicit EBS mappings to AMIs

---
 .ci/packer/aws-runners.pkr.hcl | 88 +++++++++++++++++++++++++++++++---
 1 file changed, 81 insertions(+), 7 deletions(-)

diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 0ce26ab786..93dcb778a1 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -24,6 +24,11 @@ source "amazon-ebs" "ubuntu" {
   source_ami    = "ami-0aeb7c931a5a61206"
   ssh_username  = "ubuntu"
   communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
   tags = {
     OS_Version = "Ubuntu"
     Release    = "22.04"
@@ -40,6 +45,11 @@ source "amazon-ebs" "debian-10-amd64" {
   source_ami    = "ami-0d90bed76900e679a"
   ssh_username  = "admin"
   communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
   tags = {
     OS_Version = "Debian"
     Release    = "10"
@@ -56,6 +66,11 @@ source "amazon-ebs" "debian-10-arm64" {
   source_ami    = "ami-06dac44ad759182bd"
   ssh_username  = "admin"
   communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
   tags = {
     OS_Version = "Debian"
     Release    = "10"
@@ -72,6 +87,11 @@ source "amazon-ebs" "debian-11-amd64" {
   source_ami    = "ami-0c7c4e3c6b4941f0f"
   ssh_username  = "admin"
   communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
   tags          = {
     OS_Version = "Debian"
     Release    = "11"
@@ -88,6 +108,11 @@ source "amazon-ebs" "centos-8-amd64" {
   source_ami    = "ami-045b0a05944af45c1"
   ssh_username  = "centos"
   communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
   tags          = {
     OS_Version = "Centos"
     Release    = "8"
@@ -104,6 +129,11 @@ source "amazon-ebs" "centos-8-arm64" {
   source_ami    = "ami-01cdc9e8306344fe0"
   ssh_username  = "centos"
   communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
   tags          = {
     OS_Version = "Centos"
     Release    = "8"
@@ -113,6 +143,49 @@ source "amazon-ebs" "centos-8-arm64" {
   force_deregister = local.force_deregister
 }
 
+source "amazon-ebs" "oracle-linux-8" {
+  ami_name      = "oracle-linux-8-x86-64-runner-1"
+  instance_type = "t3.xlarge"
+  region        = local.aws_region
+  source_ami    = "ami-00371eeb8fd8e0e16"
+  ssh_username  = "ec2-user"
+  communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
+  tags          = {
+    OS_Version = "Oracle Linux"
+    Release    = "8"
+    Arch       = "x86-64"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+
+}
+
+source "amazon-ebs" "sles15" {
+  ami_name      = "sles15-runner-1"
+  instance_type = "t3.xlarge"
+  region        = local.aws_region
+  source_ami    = "ami-0f7cb53c916a75006"
+  ssh_username  = "ec2-user"
+  communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 15
+    volume_type = "gp3"
+  }
+  tags          = {
+    OS_Version = "SUSE Linux Enterprise Server 15 SP3"
+    Release    = "8"
+    Arch       = "ARM64"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
 source "amazon-ebs" "windows2019" {
   ami_name        = "windows-2019-runner-1"
   instance_type   = "c5.2xlarge"
@@ -135,14 +208,15 @@ source "amazon-ebs" "windows2019" {
 build {
   name = "e2e runners AMIs"
   sources = [
-#    "source.amazon-ebs.ubuntu",
-#    "source.amazon-ebs.debian-10-amd64",
-#    "source.amazon-ebs.debian-10-arm64",
-#    "source.amazon-ebs.debian-11-amd64",
+    "source.amazon-ebs.ubuntu",
+    "source.amazon-ebs.debian-10-amd64",
+    "source.amazon-ebs.debian-10-arm64",
+    "source.amazon-ebs.debian-11-amd64",
     "source.amazon-ebs.centos-8-amd64",
-    "source.amazon-ebs.centos-8-arm64"
-#    "source.amazon-ebs.windows2019"
-
+    "source.amazon-ebs.centos-8-arm64",
+    "source.amazon-ebs.oracle-linux-8",
+    "source.amazon-ebs.sles15",
+    "source.amazon-ebs.windows2019"
   ]
 
   provisioner "ansible" {

From d92e3b4afa0884c16a0e7b57665ef0697f52298d Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 15 Nov 2022 16:40:21 +0100
Subject: [PATCH 21/78] Windows 2019 AMI with preinstalled dependencies

---
 .ci/.e2e-platforms.yaml                |  7 ++++
 .ci/.e2e-tests-tmp.yaml                | 44 ++++++---------------
 .ci/ansible/playbook.yml               |  2 +
 .ci/packer/aws-runners-windows.pkr.hcl | 55 ++++++++++++++++++++++++++
 .ci/packer/aws-runners.pkr.hcl         | 22 +----------
 .ci/packer/scripts/winrm_bootstrap.txt | 47 ----------------------
 6 files changed, 78 insertions(+), 99 deletions(-)
 create mode 100644 .ci/packer/aws-runners-windows.pkr.hcl
 delete mode 100644 .ci/packer/scripts/winrm_bootstrap.txt

diff --git a/.ci/.e2e-platforms.yaml b/.ci/.e2e-platforms.yaml
index 1f505ec853..7d234c81a2 100644
--- a/.ci/.e2e-platforms.yaml
+++ b/.ci/.e2e-platforms.yaml
@@ -108,4 +108,11 @@ PLATFORMS:
     shell_type: sh
     username: centos
     dependencies_installed: true
+  windows2019_preinstalled:
+    description: "Windows 2019 x86_64 (HVM) preinstalled"
+    image: "ami-05ed75c207e51701b"
+    instance_type: "c5.2xlarge"
+    shell_type: cmd
+    username: ogc
+    dependencies_installed: true
 
diff --git a/.ci/.e2e-tests-tmp.yaml b/.ci/.e2e-tests-tmp.yaml
index 7542ed1822..2c48b37547 100644
--- a/.ci/.e2e-tests-tmp.yaml
+++ b/.ci/.e2e-tests-tmp.yaml
@@ -1,37 +1,19 @@
 ---
 SUITES:
-  - suite: "helm"
-    provider: "docker"
-    scenarios:
-      - name: "APM Server"
-        tags: "apm-server"
-        platforms: [ "debian_10_amd64_preinstalled" ]
-      - name: "Filebeat"
-        tags: "filebeat"
-        platforms: [ "debian_10_amd64_preinstalled" ]
-      - name: "Metricbeat"
-        tags: "metricbeat"
-        platforms: [ "debian_10_amd64_preinstalled" ]
+#  - suite: "helm"
+#    provider: "docker"
+#    scenarios:
+#      - name: "APM Server"
+#        tags: "apm-server"
+#        platforms: [ "debian_10_amd64_preinstalled" ]
+#      - name: "Filebeat"
+#        tags: "filebeat"
+#        platforms: [ "debian_10_amd64_preinstalled" ]
+#      - name: "Metricbeat"
+#        tags: "metricbeat"
+#        platforms: [ "debian_10_amd64_preinstalled" ]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["centos8_arm64_preinstalled", "centos8_amd64_preinstalled"]
-#      - name: "Upgrade Agent"
-#        tags: "upgrade_agent"
-#        platforms: [ "debian_10_amd64_preinstalled" ]
-  - suite: "kubernetes-autodiscover"
-    provider: "docker"
-    scenarios:
-      - name: "kubernetes autodiscover with elastic-agent"
-        tags: "elastic-agent"
-        platforms: [ "debian_10_amd64_preinstalled" ]
-      - name: "kubernetes autodiscover with filebeat"
-        tags: "filebeat"
-        platforms: [ "debian_10_amd64_preinstalled" ]
-      - name: "kubernetes autodiscover with heartbeat"
-        tags: "heartbeat"
-        platforms: [ "debian_10_amd64_preinstalled" ]
-      - name: "kubernetes autodiscover with metricbeat"
-        tags: "metricbeat"
-        platforms: [ "debian_10_amd64_preinstalled" ]
\ No newline at end of file
+        platforms: ["windows2019_preinstalled", "centos8_amd64_preinstalled"]
diff --git a/.ci/ansible/playbook.yml b/.ci/ansible/playbook.yml
index b878783039..4ecfc37f75 100644
--- a/.ci/ansible/playbook.yml
+++ b/.ci/ansible/playbook.yml
@@ -164,6 +164,8 @@
       user: "{{ ansible_user }}"
       key: "https://github.com/{{ item }}.keys"
     with_lines: cat ./github-ssh-keys
+    when:
+      - 'ansible_os_family not in ["Windows"]'
     tags:
       - setup-node
       - setup-ami
diff --git a/.ci/packer/aws-runners-windows.pkr.hcl b/.ci/packer/aws-runners-windows.pkr.hcl
new file mode 100644
index 0000000000..c2dfb014e8
--- /dev/null
+++ b/.ci/packer/aws-runners-windows.pkr.hcl
@@ -0,0 +1,55 @@
+packer {
+  required_plugins {
+    amazon = {
+      version = ">= 1.1.5"
+      source  = "github.com/hashicorp/amazon"
+    }
+  }
+}
+
+variable "skip_create_ami" {
+  type = bool
+  default = false
+}
+
+locals {
+  aws_region = "us-east-2"
+  force_deregister = true
+}
+
+source "amazon-ebs" "windows2019" {
+  ami_name        = "windows-2019-runner-1"
+  instance_type   = "c5.2xlarge"
+  region          = local.aws_region
+  source_ami      = "ami-0587bd602f1da2f1d"
+  ssh_username  = "ogc"
+  communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 60
+    volume_type = "gp3"
+  }
+
+  tags            = {
+    OS_Version  = "Windows"
+    Release     = "2019"
+    Arch        = "AMD64"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
+build {
+  name = "e2e runners AMIs"
+  sources = [
+    "source.amazon-ebs.windows2019"
+  ]
+
+  provisioner "ansible" {
+    user = build.User
+    ansible_env_vars  =  ["PACKER_BUILD_NAME={{ build_name }}"]
+    playbook_file     = ".ci/ansible/playbook.yml"
+    extra_arguments   = ["--tags", "setup-ami", "--extra-vars", "nodeShellType=cmd"]
+    galaxy_file       = ".ci/ansible/requirements.yml"
+  }
+}
\ No newline at end of file
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 93dcb778a1..07dacf3e17 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -186,25 +186,6 @@ source "amazon-ebs" "sles15" {
   force_deregister = local.force_deregister
 }
 
-source "amazon-ebs" "windows2019" {
-  ami_name        = "windows-2019-runner-1"
-  instance_type   = "c5.2xlarge"
-  region          = local.aws_region
-  source_ami      = "ami-0587bd602f1da2f1d"
-  winrm_username  = "ogc"
-  winrm_insecure  = true
-  winrm_use_ssl   = true
-  communicator    = "winrm"
-  user_data_file  = ".ci/packer/scripts/winrm_bootstrap.txt"
-  tags            = {
-    OS_Version  = "Windows"
-    Release     = "9"
-    Arch        = "AMD64"
-  }
-  skip_create_ami  = var.skip_create_ami
-  force_deregister = local.force_deregister
-}
-
 build {
   name = "e2e runners AMIs"
   sources = [
@@ -215,8 +196,7 @@ build {
     "source.amazon-ebs.centos-8-amd64",
     "source.amazon-ebs.centos-8-arm64",
     "source.amazon-ebs.oracle-linux-8",
-    "source.amazon-ebs.sles15",
-    "source.amazon-ebs.windows2019"
+    "source.amazon-ebs.sles15"
   ]
 
   provisioner "ansible" {
diff --git a/.ci/packer/scripts/winrm_bootstrap.txt b/.ci/packer/scripts/winrm_bootstrap.txt
deleted file mode 100644
index 074b42e285..0000000000
--- a/.ci/packer/scripts/winrm_bootstrap.txt
+++ /dev/null
@@ -1,47 +0,0 @@
-<powershell>
-
-# MAKE SURE IN YOUR PACKER CONFIG TO SET:
-#
-#
-#    "winrm_username": "Administrator",
-#    "winrm_insecure": true,
-#    "winrm_use_ssl": true,
-#
-#
-
-
-write-output "Running User Data Script"
-write-host "(host) Running User Data Script"
-
-Set-ExecutionPolicy Unrestricted -Scope LocalMachine -Force -ErrorAction Ignore
-
-# Don't set this before Set-ExecutionPolicy as it throws an error
-$ErrorActionPreference = "stop"
-
-# Remove HTTP listener
-Remove-Item -Path WSMan:\Localhost\listener\listener* -Recurse
-
-# Create a self-signed certificate to let ssl work
-$Cert = New-SelfSignedCertificate -CertstoreLocation Cert:\LocalMachine\My -DnsName "packer"
-New-Item -Path WSMan:\LocalHost\Listener -Transport HTTPS -Address * -CertificateThumbPrint $Cert.Thumbprint -Force
-
-# WinRM
-write-output "Setting up WinRM"
-write-host "(host) setting up WinRM"
-
-cmd.exe /c winrm quickconfig -q
-cmd.exe /c winrm set "winrm/config" '@{MaxTimeoutms="1800000"}'
-cmd.exe /c winrm set "winrm/config/winrs" '@{MaxMemoryPerShellMB="1024"}'
-cmd.exe /c winrm set "winrm/config/service" '@{AllowUnencrypted="true"}'
-cmd.exe /c winrm set "winrm/config/client" '@{AllowUnencrypted="true"}'
-cmd.exe /c winrm set "winrm/config/service/auth" '@{Basic="true"}'
-cmd.exe /c winrm set "winrm/config/client/auth" '@{Basic="true"}'
-cmd.exe /c winrm set "winrm/config/service/auth" '@{CredSSP="true"}'
-cmd.exe /c winrm set "winrm/config/listener?Address=*+Transport=HTTPS" "@{Port=`"5986`";Hostname=`"packer`";CertificateThumbprint=`"$($Cert.Thumbprint)`"}"
-cmd.exe /c netsh advfirewall firewall set rule group="remote administration" new enable=yes
-cmd.exe /c netsh firewall add portopening TCP 5986 "Port 5986"
-cmd.exe /c net stop winrm
-cmd.exe /c sc config winrm start= auto
-cmd.exe /c net start winrm
-
-</powershell>

From af357ed781480712c8f205e3a30c47ee0bd65798 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 15 Nov 2022 17:19:38 +0100
Subject: [PATCH 22/78] Windows 2019 AMI with preinstalled dependencies: fixed
 image name

---
 .ci/.e2e-platforms.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/.e2e-platforms.yaml b/.ci/.e2e-platforms.yaml
index 7d234c81a2..54a5a8e94a 100644
--- a/.ci/.e2e-platforms.yaml
+++ b/.ci/.e2e-platforms.yaml
@@ -110,7 +110,7 @@ PLATFORMS:
     dependencies_installed: true
   windows2019_preinstalled:
     description: "Windows 2019 x86_64 (HVM) preinstalled"
-    image: "ami-05ed75c207e51701b"
+    image: "windows-2019-runner-1"
     instance_type: "c5.2xlarge"
     shell_type: cmd
     username: ogc

From 5f1920ed4ca7f91f7b95dcc181e194c4be65846d Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 15 Nov 2022 17:51:54 +0100
Subject: [PATCH 23/78] Windows 2019 AMI with preinstalled dependencies:
 removed main ebs mapping from ansible

---
 .ci/ansible/tasks/runners.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index e09c6b30ea..95e272ca9f 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -64,10 +64,10 @@
     count_tag:
       name: "e2e-{{ instanceID }}"
     volumes:
-      - device_name: /dev/sda1
-        volume_type: gp3
-        volume_size: "{{ (nodeLabel != 'windows2019') | ternary(15, 60) }}"
-        delete_on_termination: yes
+#      - device_name: /dev/sda1
+#        volume_type: gp3
+#        volume_size: "{{ (nodeLabel != 'windows2019') | ternary(15, 60) }}"
+#        delete_on_termination: yes
       - device_name: /dev/xvda
         volume_type: gp3
         volume_size: 200

From 94fd1c2a922bbf03332c5de155a81d55d6da8c02 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 30 Nov 2022 15:58:05 +0100
Subject: [PATCH 24/78] CI: Moved arbitrary functions to e2eUtils.groovy

---
 .ci/.e2e-platforms.yaml                |  23 +-
 .ci/Jenkinsfile                        | 443 +++----------------------
 .ci/Makefile                           |  13 +
 .ci/ansible/files/sshd_config          |   4 -
 .ci/e2eUtils.groovy                    | 408 +++++++++++++++++++++++
 .ci/packer/aws-runners-windows.pkr.hcl |  55 ---
 .ci/packer/aws-runners.pkr.hcl         | 191 +++++++----
 7 files changed, 611 insertions(+), 526 deletions(-)
 delete mode 100644 .ci/ansible/files/sshd_config
 create mode 100644 .ci/e2eUtils.groovy
 delete mode 100644 .ci/packer/aws-runners-windows.pkr.hcl

diff --git a/.ci/.e2e-platforms.yaml b/.ci/.e2e-platforms.yaml
index 54a5a8e94a..9754643458 100644
--- a/.ci/.e2e-platforms.yaml
+++ b/.ci/.e2e-platforms.yaml
@@ -66,51 +66,60 @@ PLATFORMS:
     instance_type: "t3.xlarge"
     shell_type: sh
     username: ubuntu
+
+  # Machines with preinstalled dependencies
+  # Argument "image" doesn't fully specify the AMI name.
+  # The result image name is concatenated with branch name and corresponds to pattern:
+  #   [image]-[branch]
+  #   for example:
+  #     - ubuntu-2204-e2e-runner-main
+  #     - ubuntu-2204-e2e-runner-8.3
+  #     - ubuntu-2204-e2e-runner-PR-3698 (a temporary AMI is being built and tested when we change ansible packer dirs)
   ubuntu_22_04_amd64_preinstalled:
     description: "Ubuntu 22.04 AMD64 Preinstalled"
-    image: "ubuntu-2204-e2e-runner-1"
+    image: "ubuntu-2204-e2e-runner"
     instance_type: "t3.xlarge"
     shell_type: sh
     username: ubuntu
     dependencies_installed: true
   debian_10_arm64_preinstalled:
     description: "Debian 10 ARM64 Preinstalled"
-    image: "debian-10-arm64-runner-1"
+    image: "debian-10-arm64-runner"
     instance_type: "a1.large"
     shell_type: sh
     username: admin
     dependencies_installed: true
   debian_10_amd64_preinstalled:
     description: "Debian 10 AMD64 Preinstalled"
-    image: "debian-10-amd64-runner-1"
+    image: "debian-10-amd64-runner"
     instance_type: "t3.xlarge"
     shell_type: sh
     username: admin
     dependencies_installed: true
   debian_11_amd64_preinstalled:
     description: "Debian 11 AMD64 Preinstalled"
-    image: "debian-11-amd64-runner-1"
+    image: "debian-11-amd64-runner"
     instance_type: "t3.xlarge"
     shell_type: sh
     username: admin
     dependencies_installed: true
   centos8_arm64_preinstalled:
     description: "CentOS Stream 8 aarch64 preinstalled"
-    image: "centos-8-arm64-runner-1"
+    image: "centos-8-arm64-runner"
     instance_type: "a1.large"
     shell_type: sh
     username: centos
     dependencies_installed: true
   centos8_amd64_preinstalled:
     description: "CentOS Stream 8 x86_64"
-    image: "centos-8-amd64-runner-1"
+    image: "centos-8-amd64-runner"
     instance_type: "t3.xlarge"
     shell_type: sh
     username: centos
     dependencies_installed: true
   windows2019_preinstalled:
     description: "Windows 2019 x86_64 (HVM) preinstalled"
-    image: "windows-2019-runner-1"
+    image: "windows-2019-runner"
     instance_type: "c5.2xlarge"
     shell_type: cmd
     username: ogc
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 0287b64478..20a9cf5ad4 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -27,9 +27,12 @@ pipeline {
     ELASTIC_CLOUD_SECRET = 'secret/observability-team/ci/elastic-cloud/observability-team-user'
     GCP_PROVISIONER_SECRET = 'secret/observability-team/ci/service-account/jenkins-gce-provisioner'
     AWS_PROVISIONER_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
-    TEST_MATRIX_FILE = "${params.testMatrixFile}"
     E2E_SSH_KEY = "${env.REAL_BASE_DIR}/e2essh"
     E2E_SSH_KEY_PUB = "${env.E2E_SSH_KEY}.pub"
+    TEST_MATRIX_FILE = "${params.testMatrixFile}"
+    RUN_AS_MAIN_BRANCH = "${params.Run_As_Main_Branch}"
+    DESTROY_CLOUD_RESOURCES = "${params.DESTROY_CLOUD_RESOURCES}"
+    RUN_TEST_SUITES = "${params.runTestsSuites}"
   }
   options {
     timeout(time: 120, unit: 'MINUTES')
@@ -70,7 +73,7 @@ pipeline {
     string(name: 'GITHUB_CHECK_SHA1', defaultValue: '', description: 'Git SHA for the upstream project (branch or PR)')
   }
   stages {
-    stage('Initializing'){
+    stage('Initializing') {
       options { skipDefaultCheckout() }
       environment {
         HOME = "${env.WORKSPACE}"
@@ -91,6 +94,7 @@ pipeline {
         LOG_LEVEL = "${params.LOG_LEVEL.trim()}"
         TIMEOUT_FACTOR = "${params.TIMEOUT_FACTOR.trim()}"
         GITHUB_CHECK_REPO = "${params.GITHUB_CHECK_REPO.trim()}"
+        GITHUB_CHECK_NAME = "${params.GITHUB_CHECK_NAME.trim()}"
         GITHUB_CHECK_SHA1 = "${params.GITHUB_CHECK_SHA1.trim()}"
       }
       stages {
@@ -98,12 +102,13 @@ pipeline {
           steps {
             pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
             deleteDir()
-            gitCheckout(basedir: BASE_DIR, githubNotifyFirstTimeContributor: true)
-            githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
+            gitCheckout(basedir: BASE_DIR, githubNotifyFirstTimeContributor: true) 
             stash allowEmpty: true, name: 'source', useDefaultExcludes: false
             setEnvVar("GO_VERSION", readFile("${env.REAL_BASE_DIR}/.go-version").trim())
             setEnvVar("LABELS_STRING", "buildURL=${env.BUILD_URL} gitSha=${env.GIT_BASE_COMMIT} build=${env.BUILD_ID} repo=${env.REPO} branch=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')} type=ci")
-            checkSkipTests()
+            def e2eUtils = load ".ci/e2eUtils.groovy"
+            e2eUtils.checkSkipTests()
+            e2eUtils.githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
           }
         }
         stage('Build Docs') {
@@ -135,6 +140,9 @@ pipeline {
           environment {
             GO111MODULE = 'on'
             PATH = "${env.HOME}/bin:${env.REAL_BASE_DIR}/bin:${HOME}/go/bin:${env.PATH}"
+            SUITE = 'fleet'
+            STACK_INSTANCE_ID = "${env.BUILD_URL}_stack"
+            TAGS = "non-existing-tag"
           }
           when {
             beforeAgent true
@@ -147,28 +155,21 @@ pipeline {
             withGithubNotify(context: 'Deploy Stack', tab: 'tests') {
               deleteDir()
               unstash 'source'
-              script {
-                // Deploy the test infrastructure
+              script {                
                 sh "ssh-keygen -b 4096 -t rsa -f ${E2E_SSH_KEY} -q -N \"\" "
 
                 dir("${env.REAL_BASE_DIR}") {
-                  withEnv([
-                    "SUITE=fleet",
-                    "STACK_INSTANCE_ID=${env.BUILD_URL}_stack",
-                    "TAGS=non-existing-tag"
-                  ]) {
-                    ciBuild() {
-                      sh(label: 'Create Stack node', script: "make -C .ci provision-stack")
-                    }
-                    ciBuild() {
-                      retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Setup Stack node', script: "make -C .ci setup-stack")
-                      }
+                  e2eUtils.ciBuild() {
+                    sh(label: 'Create Stack node', script: "make -C .ci provision-stack")
+                  }
+                  e2eUtils.ciBuild() {
+                    retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                      sh(label: 'Setup Stack node', script: "make -C .ci setup-stack")
                     }
-                    ciBuild() {
-                      retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Start Elastic Stack', script: "make -C .ci start-elastic-stack")
-                      }
+                  }
+                  e2eUtils.ciBuild() {
+                    retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                      sh(label: 'Start Elastic Stack', script: "make -C .ci start-elastic-stack")
                     }
                   }
 
@@ -196,24 +197,16 @@ pipeline {
           }
           steps {
             withGithubNotify(context: 'E2E Tests', tab: 'tests') {
-              script {
-                def suitesParam = params.runTestsSuites
+              script {                
                 def existingSuites = readYaml(file: "${env.REAL_BASE_DIR}/${TEST_MATRIX_FILE}")
-                def parallelTasks = [:]
-                if (suitesParam == "") {
-                  log(level: 'DEBUG', text: "Iterate through existing test suites")
-                  existingSuites['SUITES'].each { item ->
-                    checkTestSuite(parallelTasks, item)
-                  }
-                } else {
-                  log(level: 'DEBUG', text: "Iterate through the comma-separated test suites (${suitesParam}), comparing with the existing test suites")
-                  suitesParam.split(',').each { suiteParam ->
-                    existingSuites['SUITES'].findAll { suiteParam.trim() == it.suite }.each { item ->
-                      checkTestSuite(parallelTasks, item)
-                    }
-                  }
-                }
-                parallel(parallelTasks)
+                runE2ETests(
+                  testMatrix: existingSuites,
+                  selectedSuites: env.RUN_TEST_SUITES,
+                  runAsMainBranch: env.RUN_AS_MAIN_BRANCH,
+                  githubCheckSha1: env.GITHUB_CHECK_SHA1,
+                  githubCheckRepo: env.GITHUB_CHECK_REP
+                  destroyTestRunner: env.DESTROY_CLOUD_RESOURCES
+                )
               }
             }
           }
@@ -221,19 +214,7 @@ pipeline {
             cleanup {
               // Once all tests are complete we need to teardown the single instance with the deployed stack
               script {
-                def stackMachine = getMachineInfo('stack')
-                if (!params.DESTROY_CLOUD_RESOURCES) {
-                  def stackRunnerIP = getNodeIp('stack')
-                  log(level: 'DEBUG', text: "Stack instance won't be destroyed after the build. Please SSH into the stack machine on ${stackRunnerIP}")
-                } else {
-                  dir("${env.REAL_BASE_DIR}") {
-                    ciBuild() {
-                      retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Destroy stack node', script: "make -C .ci destroy-stack")
-                      }
-                    }
-                  }
-                }
+                destroyStack(destroyCloudResources: env.DESTROY_CLOUD_RESOURCES)
               }
             }
           }
@@ -262,358 +243,12 @@ pipeline {
   }
   post {
     cleanup {
-      doNotifyBuildResult(params.notifyOnGreenBuilds)
-    }
-  }
-}
-
-// this function evaluates if the test stage of the build must be executed
-def checkSkipTests() {
-  dir("${BASE_DIR}"){
-
-    // only docs means no tests are run
-    if (isGitRegionMatch(patterns: [ '.*\\.md' ], shouldMatchAll: true)) {
-      setEnvVar("SKIP_TESTS", true)
-      return
-    }
-
-    // patterns for all places that should trigger a full build
-    def regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/helm/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
-    setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: regexps, shouldMatchAll: false))
-  }
-}
-
-/*
- * Runs the Make build at the CI, executing the closure in the context of Ansible + AWS
- */
-def ciBuild(Closure body){
-  withEnv([
-    "SSH_KEY=${E2E_SSH_KEY}"
-  ]) {
-    def awsProps = getVaultSecret(secret: "${AWS_PROVISIONER_SECRET}")
-    def awsAuthObj = awsProps?.data
-    withEnv([
-      "ANSIBLE_CONFIG=${env.REAL_BASE_DIR}/.ci/ansible/ansible.cfg",
-      "ANSIBLE_HOST_KEY_CHECKING=False",
-    ]){
-      withVaultToken(){
-        withEnvMask(vars: [
-          [var: "AWS_ACCESS_KEY_ID", password: awsAuthObj.access_key],
-          [var: "AWS_SECRET_ACCESS_KEY", password: awsAuthObj.secret_key]
-        ]) {
-          withOtelEnv() {
-            retryWithSleep(retries: 3, seconds: 5, backoff: true){
-              sh("make -C .ci setup-env") // make sure the environment is created
-            }
-            body()
-          }
-        }
-      }
-    }
-  }
-}
-
-def getNodeIp(nodeType){
-  return sh(label: "Get IP address of the ${nodeType}", script: "cat ${REAL_BASE_DIR}/.ci/.${nodeType}-host-ip", returnStdout: true)
-}
-
-def getRemoteE2EPath(testRunner, platform) {
-  if (platform.contains("windows")) {
-    return "C:\\Users\\${testRunner.user}\\e2e-testing\\"
-  }
-
-  return "/home/${testRunner.user}/e2e-testing/"
-}
-
-def getMachineInfo(platform){
-  def machineYaml = readYaml(file: "${env.REAL_BASE_DIR}/.ci/.e2e-platforms.yaml")
-  def machines = machineYaml['PLATFORMS']
-  log(level: 'INFO', text: "getMachineInfo: machines.get(platform)=${machines.get(platform)}")
-  return machines.get(platform)
-}
-
-def checkTestSuite(Map parallelTasks = [:], Map item = [:]) {
-  def suite = item.suite
-  def platforms = item.platforms
-
-  // Predefine the remote provider to use the already provisioned stack VM.
-  // Each suite or scenario in the CI test suite would be able to define its own provider
-  // (i.e. docker). If empty, remote will be used as fallback
-  def suiteProvider = item.provider
-  if (!suiteProvider || suiteProvider?.trim() == '') {
-    suiteProvider = 'remote'
-  }
-
-  item.scenarios.each { scenario ->
-    def name = scenario.name
-    def platformsValue = platforms
-
-    def scenarioProvider = scenario.provider
-    // if the scenario does not set its own provider, use suite's provider
-    if (!scenarioProvider || scenarioProvider?.trim() == '') {
-      scenarioProvider = suiteProvider
-    }
-
-    def scenarioPlatforms = scenario.platforms
-    if (scenarioPlatforms?.size() > 0) {
-      // scenario platforms take precedence over suite platforms, overriding them
-      platformsValue = scenarioPlatforms
-    }
-    def pullRequestFilter = scenario.containsKey('pullRequestFilter') ? scenario.pullRequestFilter : ''
-    def tags = scenario.tags
-      platformsValue.each { rawPlatform ->
-        // platform is not space based, so let's ensure no extra spaces can cause misbehaviours.
-        def platform = rawPlatform.trim()
-        log(level: 'INFO', text: "Adding ${suite}:${platform}:${tags} test suite to the build execution")
-        def machineInfo = getMachineInfo(platform)
-        def stageName = "${suite}_${platform}_${tags}"
-        parallelTasks["${stageName}"] = generateFunctionalTestStep(name: "${name}",
-                                                                  platform: platform,
-                                                                  provider: scenarioProvider,
-                                                                  suite: "${suite}",
-                                                                  tags: "${tags}",
-                                                                  pullRequestFilter: "${pullRequestFilter}",
-                                                                  machine: machineInfo,
-                                                                  stageName: stageName)
-      }
-  }
-}
-
-/*
- * Sends out notification of the build result to Slack
- */
-def doNotifyBuildResult(boolean slackNotify) {
-  def doSlackNotify = true // always try to notify on failures
-  def githubCheckStatus = 'FAILURE'
-  if (currentBuild.currentResult == 'SUCCESS') {
-    githubCheckStatus = 'SUCCESS'
-    doSlackNotify = slackNotify // if the build status is success, read the parameter
-  }
-
-  githubCheckNotify(githubCheckStatus)
-
-
-  def testsSuites = "${params.runTestsSuites}"
-  if (testsSuites?.trim() == "") {
-    testsSuites = "All suites"
-  }
-
-  def channels = "${env.SLACK_CHANNEL}"
-  if (channels?.trim() == "") {
-    channels = "observablt-bots"
-  }
-
-  def header = "*Test Suite*: " + testsSuites
-  notifyBuildResult(analyzeFlakey: true,
-                    jobName: getFlakyJobName(withBranch: "${env.JOB_BASE_NAME}"),
-                    prComment: true,
-                    slackHeader: header,
-                    slackChannel: "${channels}",
-                    slackComment: true,
-                    slackNotify: doSlackNotify)
-}
-
-/**
- Notify the GitHub check of the parent stream
- **/
-def githubCheckNotify(String status) {
-  if (params.GITHUB_CHECK_NAME?.trim() && params.GITHUB_CHECK_REPO?.trim() && params.GITHUB_CHECK_SHA1?.trim()) {
-    githubNotify context: "${params.GITHUB_CHECK_NAME}",
-      description: "${params.GITHUB_CHECK_NAME} ${status.toLowerCase()}",
-      status: "${status}",
-      targetUrl: "${env.RUN_DISPLAY_URL}",
-      sha: params.GITHUB_CHECK_SHA1, account: 'elastic', repo: params.GITHUB_CHECK_REPO, credentialsId: env.JOB_GIT_CREDENTIALS
-  }
-}
-
-def generateFunctionalTestStep(Map args = [:]){
-  def name = args.get('name')
-  def name_normalize = name.replace(' ', '_')
-  def platform = args.get('platform')
-  def provider = args.get('provider')
-  def suite = args.get('suite')
-  def tags = args.get('tags')
-  def pullRequestFilter = args.get('pullRequestFilter')?.trim() ? args.get('pullRequestFilter') : ''
-  def machine = args.get('machine')
-  def stageName = args.get('stageName')
-
-  // TODO: Is this still relevant?
-  if (isPR() || isUpstreamTrigger(filter: 'PR-')) {
-    // when the "Run_As_Main_Branch" param is disabled, we will honour the PR filters, which
-    // basically exclude some less frequent platforms or operative systems. If the user enabled
-    // this param, the pipeline will remove the filters from the test runner.
-    if (!params.Run_As_Main_Branch) {
-      tags += pullRequestFilter
-    }
-  }
-
-  def goArch = "amd64"
-  if (platform.contains("arm64")) {
-    goArch = "arm64"
-  }
-
-  // sanitize tags to create the file
-  def sanitisedTags = tags.replaceAll("\\s","_")
-  sanitisedTags = sanitisedTags.replaceAll("~","")
-  sanitisedTags = sanitisedTags.replaceAll("@","")
-
-  def githubCheckSha1 = params.GITHUB_CHECK_SHA1?.trim() ? params.GITHUB_CHECK_SHA1 : ''
-  def githubCheckRepo = params.GITHUB_CHECK_REPO?.trim() ? params.GITHUB_CHECK_REPO : ''
-
-  // Setup environment for platform
-  def envContext = []
-  envContext.add("PROVIDER=${provider}")
-  envContext.add("GITHUB_CHECK_SHA1=${githubCheckSha1}")
-  envContext.add("GITHUB_CHECK_REPO=${githubCheckRepo}")
-  envContext.add("SUITE=${suite}")
-  envContext.add("TAGS=${tags}")
-  envContext.add("REPORT_PREFIX=${suite}_${platform}_${sanitisedTags}")
-  envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},elastic_agent_version=${env.ELASTIC_AGENT_VERSION},stack_version=${env.STACK_VERSION}")
-  // VM characteristics
-  envContext.add("NODE_LABEL=${platform}")
-  envContext.add("NODE_IMAGE=${machine.image}")
-  envContext.add("NODE_INSTANCE_ID=${env.BUILD_URL}_${platform}_${suite}_${tags}")
-  envContext.add("NODE_INSTANCE_TYPE=${machine.instance_type}")
-  envContext.add("NODE_SHELL_TYPE=${machine.shell_type}")
-  envContext.add("NODE_USER=${machine.username}")
-
-  return {
-    // Set the worker as flaky for the time being, this will be changed in the finally closure.
-    setFlakyWorker(stageName)
-    retryWithNode(labels: 'ubuntu-20.04 && gobld/machineType:e2-small', forceWorkspace: true, forceWorker: true, stageName: stageName){
-      try {
-        deleteDir()
-        dir("${env.REAL_BASE_DIR}") {
-          unstash 'sourceEnvModified'
-          withEnv(envContext) {
-            // This step will help to send the APM traces to the
-            // withOtelEnv is the one that uses the APM service defined by the Otel Jenkins plugin.
-            // withAPMEnv uses Vault to prepare the context.
-            // IMPORTANT: withAPMEnv is now the one in used since withOtelEnv uses a specific Opentelemetry Collector at the moment.
-            // TODO: This will need to be integrated into the provisioned VMs
-            withAPMEnv() {
-              // we are separating the different test phases to avoid recreating
-              ciBuild() {
-                  sh(label: 'Start node', script: "make -C .ci provision-node")
-              }
-
-              // make goal to run the tests, which is platform-dependant
-              def runCommand = "run-tests"
-
-              if (platform.contains("windows")) {
-                runCommand = "run-tests-win"
-                // Ansible wait_for module is not enough to mitigate the timeout
-                log(level: 'DEBUG', text: "Sleeping 300 seconds on Windows so that SSH is accessible in the remote instance.")
-                sleep(300)
-              }
-              if (!machine.dependencies_installed) {
-                ciBuild() {
-                  retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                    sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
-                  }
-                }
-              }
-              ciBuild() {
-                sh(label: 'Run tests in the node', script: "make -C .ci ${runCommand}")
-              }
-            }
-          }
-        }
-      } finally {
-        withEnv(envContext) {
-          dir("${env.REAL_BASE_DIR}") {
-            // If it reaches this point then the CI worker is most likely behaving correctly
-            // there is still a chance things might fail afterwards, but this is just the finally
-            // section so we could say we are good to go.
-            // It runs after dir so if the worker is gone the an error will be thrown regarding
-            // the dir cannot be accessed in the existing none worker.
-            unsetFlakyWorker(stageName)
-            def testRunnerIP = getNodeIp("node")
-            sh "mkdir -p outputs/${testRunnerIP} || true"
-            ciBuild() {
-              sh(label: 'Fetch tests reports from node', script: "make -C .ci fetch-test-reports")
-            }
-            sh "ls -l outputs/${testRunnerIP}"
-            if (!params.DESTROY_CLOUD_RESOURCES) {
-              log(level: 'INFO', text: "Cloud instance won't be destroyed after the build. Please SSH into the test runner machine on ${testRunnerIP}.")
-            } else {
-              log(level: 'INFO', text: "Destroying Cloud instance")
-              ciBuild() {
-                retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                  sh(label: 'Destroy node', script: "make -C .ci destroy-node")
-                }
-              }
-            }
-            withEnv([
-              "ARCHITECTURE=${goArch}",
-              "CUCUMBER_REPORTS_PATH=${env.REAL_BASE_DIR}/outputs/${testRunnerIP}",
-              "PLATFORM=${platform}",
-              "SUITE=${suite}",
-              "TAGS=${tags}",
-            ]){
-              retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-                sh(script: ".ci/scripts/generate-cucumber-reports.sh", label: "generate-cucumber-reports.sh")
-              }
-            }
-            junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/${testRunnerIP}/TEST-*.xml")
-            archiveArtifacts allowEmptyArchive: true,
-              artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html"
-          }
-        }
-      }
-    }
-  }
-}
-
-def retryWithNode(Map args = [:], Closure body) {
-  try {
-    incrementRetries(args.stageName)
-    withNode(args){
-      body()
+      e2eUtils.doNotifyBuildResult(
+        slackNotify: params.notifyOnGreenBuilds,
+        slackChannel: env.SLACK_CHANNEL,
+        runTestsSuites: env.RUN_TEST_SUITES
+      )
     }
-  } catch (err) {
-    log(level: 'WARN', text: "Stage '${args.stageName}' failed, let's analyse if it's a flaky CI worker.")
-    if (isFlakyWorker(args.stageName) && isRetryAvailable(args.stageName)) {
-      log(level: 'INFO', text: "Rerun '${args.stageName}' in a new worker.")
-      retryWithNode(args) {
-        body()
-      }
-    } else {
-      error("Error '${err.toString()}'")
-    }
-  }
-}
-
-def isFlakyWorker(stageName) {
-  if (workersStatus.containsKey(stageName)) {
-    return !workersStatus.get(stageName).get('status', true)
-  }
-  return false
-}
-
-def isRetryAvailable(stageName) {
-  return workersStatus.get(stageName).get('retries', 2) < 2
-}
-
-def incrementRetries(stageName) {
-  if (workersStatus.containsKey(stageName)) {
-    def current = workersStatus[stageName].get('retries', 0)
-    workersStatus[stageName].retries = current + 1
-  } else {
-    setFlakyWorker(stageName)
-    workersStatus[stageName].retries = 1
   }
 }
 
-def setFlakyWorker(stageName) {
-  if (workersStatus.containsKey(stageName)) {
-    workersStatus[stageName].status = false
-  } else {
-    workersStatus[stageName] = [ status: false ]
-  }
-}
-
-def unsetFlakyWorker(stageName) {
-  workersStatus[stageName].status = true
-}
diff --git a/.ci/Makefile b/.ci/Makefile
index 5f93ba2e94..d6fe8900a0 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -41,6 +41,10 @@ SUITE ?= fleet
 # Tags to run. Please check out the feature files
 TAGS ?= fleet_mode
 
+# The suffix to differeciate temporaty and production AMIs that we use for EC2 instances to run tests on
+# used in .ci/packer/aws-runners.pkr.hcl
+AMI_SUFFIX ?= main
+
 # Check that given variables are set and all have non-empty values,
 # die with an error otherwise.
 #
@@ -241,3 +245,12 @@ run-tests-win: set-env-$(NODE_LABEL)
 		-t run-tests \
 		-i $(NODE_IP_ADDRESS),
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $${NODE_USER}@$(NODE_IP_ADDRESS) "powershell \"C:/Users/$${NODE_USER}/e2e-testing/.ci/scripts/functional-test.ps1\""
+
+.PHONY: validate-packer-template
+validate-packer-template:	
+	packer validate $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
+
+.PHONY: validate-packer-template
+build-amis-$(AMI_SUFFIX): validate-packer-template
+	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)	
+	packer build -var "ami_suffix=${AMI_SUFFIX}" $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
diff --git a/.ci/ansible/files/sshd_config b/.ci/ansible/files/sshd_config
deleted file mode 100644
index ed201c75ac..0000000000
--- a/.ci/ansible/files/sshd_config
+++ /dev/null
@@ -1,4 +0,0 @@
-ClientAliveInterval 30
-ClientAliveCountMax 100
-ServerAliveInterval 30
-ServerAliveCountMax 100
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
new file mode 100644
index 0000000000..b7c7cd37ef
--- /dev/null
+++ b/.ci/e2eUtils.groovy
@@ -0,0 +1,408 @@
+def runE2ETests(Map args = [:]) {
+    def parallelTasks = [:]
+    if (args.selectedSuites?.trim(0)) {
+        log(level: 'DEBUG', text: "Iterate through existing test suites")
+        args.testMatrix['SUITES'].each { item ->
+            parallelTasks += convertSuiteToTasks(
+                item: item,
+                githubCheckSha1: args.githubCheckSha1 ?: '',
+                githubCheckRepo: args.githubCheckRepo ?: '',
+                runAsMainBranch: args.runAsMainBranch,
+                amiSuffix: args.amiSuffix ?: 'main',
+                destroyTestRunner: args.destroyTestRunner ?: false
+            )
+        }
+    } else {
+        log(level: 'DEBUG', text: "Iterate through the comma-separated test suites (${args.selectedSuites}), comparing with the existing test suites")
+        args.selectedSuites.split(',').each { selectedSuite ->
+            args.testMatrix['SUITES'].findAll { selectedSuite.trim() == it.suite }.each { item ->
+                parallelTasks += convertSuiteToTasks(
+                    item: item,
+                    githubCheckSha1: args.githubCheckSha1 ?: '',
+                    githubCheckRepo: args.githubCheckRepo ?: '',
+                    runAsMainBranch: args.runAsMainBranch,
+                    amiSuffix: args.amiSuffix ?: 'main',
+                    destroyTestRunner: args.destroyTestRunner ?: false
+                )
+            }
+        }
+    }
+    parallel(parallelTasks)
+}
+
+def convertSuiteToTasks(Map args = [:]) {
+    def parallelTasks = [:]
+    def suite = args.item.suite
+    def platforms = args.item.platforms
+
+    // Predefine the remote provider to use the already provisioned stack VM.
+    // Each suite or scenario in the CI test suite would be able to define its own provider
+    // (i.e. docker). If empty, remote will be used as fallback
+    def suiteProvider = args.item.provider
+    if (!suiteProvider || suiteProvider?.trim() == '') {
+        suiteProvider = 'remote'
+    }
+
+    args.item.scenarios.each { scenario ->
+        def name = scenario.name
+        def platformsValue = platforms
+
+        def scenarioProvider = scenario.provider
+        // if the scenario does not set its own provider, use suite's provider
+        if (scenarioProvider?.trim() == '') {
+            scenarioProvider = suiteProvider
+        }
+        
+        if (scenario.platforms?.size() > 0) {
+            // scenario platforms take precedence over suite platforms, overriding them
+            platformsValue = scenario.platforms
+        }
+        def pullRequestFilter = scenario.containsKey('pullRequestFilter') ? scenario.pullRequestFilter : ''
+        def tags = scenario.tags
+        platformsValue.each { rawPlatform ->
+            // platform is not space based, so let's ensure no extra spaces can cause misbehaviours.
+            def platform = rawPlatform.trim()
+            log(level: 'INFO', text: "Adding ${suite}:${platform}:${tags} test suite to the build execution")
+            def machineInfo = getMachineInfo(platform)
+            def stageName = "${suite}_${platform}_${tags}"
+            parallelTasks["${stageName}"] = generateFunctionalTestStep(
+                name: "${name}",
+                platform: platform,
+                provider: scenarioProvider,
+                suite: "${suite}",
+                tags: "${tags}",
+                pullRequestFilter: "${pullRequestFilter}",
+                machine: machineInfo,
+                stageName: stageName,
+                runAsMainBranch: args.runAsMainBranch,
+                amiSuffix: args.amiSuffix,
+                githubCheckSha1: args.githubCheckSha1,
+                githubCheckRepo: args.githubCheckRepo,
+                destroyTestRunner: args.destroyTestRunner                    
+            )
+        }
+    }
+    return parallelTasks
+}
+
+def checkRebuildAmis() {
+    dir("${BASE_DIR}") {
+        def ami_regexps = [ "^.ci/ansible/.*", "^.ci/packer/.*"]
+        setEnvVar("REBUILD_AMIS", isGitRegionMatch(patterns: tests_regexps, shouldMatchAll: false))
+    }
+}
+
+// this function evaluates whether the test and AMIs stages must be executed
+def checkSkipTests() {
+    dir("${BASE_DIR}") {
+
+        // if only docs changed means no tests are run
+        if (isGitRegionMatch(patterns: [ '.*\\.md' ], shouldMatchAll: true)) {
+            setEnvVar("SKIP_TESTS", true)            
+            return
+        }
+
+        // patterns for all places that should trigger a full build
+        def tests_regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/helm/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
+        // def ami_regexps = [ "^.ci/ansible/.*", "^.ci/packer/.*"]
+        setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: tests_regexps, shouldMatchAll: false))
+        
+    }
+}
+
+/*
+ * Runs the Make build at the CI, executing the closure in the context of Ansible + AWS
+ */
+def ciBuild(Closure body){
+    withEnv([
+            "SSH_KEY=${E2E_SSH_KEY}"
+    ]) {
+        def awsProps = getVaultSecret(secret: "${AWS_PROVISIONER_SECRET}")
+        def awsAuthObj = awsProps?.data
+        withEnv([
+                "ANSIBLE_CONFIG=${env.REAL_BASE_DIR}/.ci/ansible/ansible.cfg",
+                "ANSIBLE_HOST_KEY_CHECKING=False",
+        ]){
+            withVaultToken(){
+                withEnvMask(vars: [
+                        [var: "AWS_ACCESS_KEY_ID", password: awsAuthObj.access_key],
+                        [var: "AWS_SECRET_ACCESS_KEY", password: awsAuthObj.secret_key]
+                ]) {
+                    withOtelEnv() {
+                        retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                            sh("make -C .ci setup-env") // make sure the environment is created
+                        }
+                        body()
+                    }
+                }
+            }
+        }
+    }
+}
+
+def getNodeIp(nodeType){
+    return sh(label: "Get IP address of the ${nodeType}", script: "cat ${REAL_BASE_DIR}/.ci/.${nodeType}-host-ip", returnStdout: true)
+}
+
+def getMachineInfo(platform){
+    def machineYaml = readYaml(file: "${env.REAL_BASE_DIR}/.ci/.e2e-platforms.yaml")
+    def machines = machineYaml['PLATFORMS']
+    log(level: 'INFO', text: "getMachineInfo: machines.get(platform)=${machines.get(platform)}")
+    return machines.get(platform)
+}
+
+/*
+ * Sends out notification of the build result to Slack
+ */
+def doNotifyBuildResult(Map args = [:]) {
+    def doSlackNotify = true // always try to notify on failures
+    def githubCheckStatus = 'FAILURE'
+    if (currentBuild.currentResult == 'SUCCESS') {
+        githubCheckStatus = 'SUCCESS'
+        doSlackNotify = args.slackNotify // if the build status is success, read the parameter
+    }
+
+    githubCheckNotify(githubCheckStatus)
+
+    def testsSuites = args.runTestsSuites?.trim() ?: "All suites"    
+    def channels = args.slackChannel?.trim() ?: "observablt-bots"    
+
+    def header = "*Test Suite*: ${testsSuites}"
+    notifyBuildResult(analyzeFlakey: true,
+            jobName: getFlakyJobName(withBranch: "${env.JOB_BASE_NAME}"),
+            prComment: true,
+            slackHeader: header,
+            slackChannel: "${channels}",
+            slackComment: true,
+            slackNotify: doSlackNotify)
+}
+
+/**
+ Notify the GitHub check of the parent stream
+ **/
+def githubCheckNotify(Map args = [:] String status, String githubCheckName, String githubCheckRepo, String githubCheckSha1) {
+    if (args.githubCheckName?.trim() && args.githubCheckRepo?.trim() && args.githubCheckSha1?.trim()) {
+        githubNotify context: "${args.githubCheckName}",
+                description: "${args.githubCheckName} ${args.status?.toLowerCase()}",
+                status: "${args.status}",
+                targetUrl: "${env.RUN_DISPLAY_URL}",
+                sha: "${args.githubCheckSha1}", 
+                account: 'elastic', 
+                repo: args.githubCheckRepo, 
+                credentialsId: env.JOB_GIT_CREDENTIALS
+    }
+}
+/**
+* 
+*
+*/
+def generateFunctionalTestStep(Map args = [:]) {
+    def name = args.get('name')
+    def name_normalize = name.replace(' ', '_')
+    def platform = args.get('platform')
+    def provider = args.get('provider')
+    def suite = args.get('suite')
+    def tags = args.get('tags')
+    def pullRequestFilter = args.get('pullRequestFilter')?.trim() ?: ''
+    def machine = args.get('machine')
+    def stageName = args.get('stageName')
+    //TODO
+    def ami_suffix = args.amiSuffix.trim() ?: 'main'
+    def runAsMainBranch = args.runAsMainBranch ?: false
+    def destroyTestRunner = args.destroyTestRunner ?: false
+    
+
+    if (isPR() || isUpstreamTrigger(filter: 'PR-')) {
+        // when the "Run_As_Main_Branch" param is disabled, we will honour the PR filters, which
+        // basically exclude some less frequent platforms or operative systems. If the user enabled
+        // this param, the pipeline will remove the filters from the test runner.
+        if (!runAsMainBranch) {
+            tags += pullRequestFilter
+        }        
+    }
+
+    def goArch = platform.contains("arm64") ? "arm64" : "amd64"
+
+    // sanitize tags to create the file
+    def sanitisedTags = tags.replaceAll("\\s","_")
+    sanitisedTags = sanitisedTags.replaceAll("~","")
+    sanitisedTags = sanitisedTags.replaceAll("@","")
+
+    def githubCheckSha1 = args.githubCheckSha1?.trim() ?: ''
+    def githubCheckRepo = args.githubCheckRepo?.trim() ?: ''
+
+    // Setup environment for platform
+    def envContext = []
+    envContext.add("PROVIDER=${provider}")
+    envContext.add("GITHUB_CHECK_SHA1=${githubCheckSha1}")
+    envContext.add("GITHUB_CHECK_REPO=${githubCheckRepo}")
+    envContext.add("SUITE=${suite}")
+    envContext.add("TAGS=${tags}")
+    envContext.add("REPORT_PREFIX=${suite}_${platform}_${sanitisedTags}")
+    envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},elastic_agent_version=${env.ELASTIC_AGENT_VERSION},stack_version=${env.STACK_VERSION}")
+    // VM characteristics
+    envContext.add("NODE_LABEL=${platform}")
+    envContext.add("NODE_IMAGE=${machine.image}-${ami_suffix}")
+    envContext.add("NODE_INSTANCE_ID=${env.BUILD_URL}_${platform}_${suite}_${tags}")
+    envContext.add("NODE_INSTANCE_TYPE=${machine.instance_type}")
+    envContext.add("NODE_SHELL_TYPE=${machine.shell_type}")
+    envContext.add("NODE_USER=${machine.username}")
+
+    return {
+        // Set the worker as flaky for the time being, this will be changed in the finally closure.
+        setFlakyWorker(stageName)
+        retryWithNode(labels: 'ubuntu-20.04 && gobld/machineType:e2-small', forceWorkspace: true, forceWorker: true, stageName: stageName){
+            try {
+                deleteDir()
+                dir("${env.REAL_BASE_DIR}") {
+                    unstash 'sourceEnvModified'
+                    withEnv(envContext) {
+                        // This step will help to send the APM traces to the
+                        // withOtelEnv is the one that uses the APM service defined by the Otel Jenkins plugin.
+                        // withAPMEnv uses Vault to prepare the context.
+                        // IMPORTANT: withAPMEnv is now the one in used since withOtelEnv uses a specific Opentelemetry Collector at the moment.
+                        // TODO: This will need to be integrated into the provisioned VMs
+                        withAPMEnv() {
+                            // we are separating the different test phases to avoid recreating
+                            ciBuild() {
+                                sh(label: 'Start node', script: "make -C .ci provision-node")
+                            }
+
+                            // make goal to run the tests, which is platform-dependant
+                            def runCommand = "run-tests"
+
+                            if (platform.contains("windows")) {
+                                runCommand = "run-tests-win"
+                                // Ansible wait_for module is not enough to mitigate the timeout
+                                log(level: 'DEBUG', text: "Sleeping 300 seconds on Windows so that SSH is accessible in the remote instance.")
+                                sleep(300)
+                            }
+                            if (!machine.dependencies_installed) {
+                                ciBuild() {
+                                    retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                                        sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
+                                    }
+                                }
+                            }
+                            ciBuild() {
+                                sh(label: 'Run tests in the node', script: "make -C .ci ${runCommand}")
+                            }
+                        }
+                    }
+                }
+            } finally {
+                withEnv(envContext) {
+                    dir("${env.REAL_BASE_DIR}") {
+                        // If it reaches this point then the CI worker is most likely behaving correctly
+                        // there is still a chance things might fail afterwards, but this is just the finally
+                        // section so we could say we are good to go.
+                        // It runs after dir so if the worker is gone the an error will be thrown regarding
+                        // the dir cannot be accessed in the existing none worker.
+                        unsetFlakyWorker(stageName)
+                        def testRunnerIP = getNodeIp("node")
+                        sh "mkdir -p outputs/${testRunnerIP} || true"
+                        ciBuild() {
+                            sh(label: 'Fetch tests reports from node', script: "make -C .ci fetch-test-reports")
+                        }
+                        sh "ls -l outputs/${testRunnerIP}"
+                        if (!destroyTestRunner) {
+                            log(level: 'INFO', text: "Cloud instance won't be destroyed after the build. Please SSH into the test runner machine on ${testRunnerIP}.")
+                        } else {
+                            log(level: 'INFO', text: "Destroying Cloud instance")
+                            ciBuild() {
+                                retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                                    sh(label: 'Destroy node', script: "make -C .ci destroy-node")
+                                }
+                            }
+                        }
+                        withEnv([
+                                "ARCHITECTURE=${goArch}",
+                                "CUCUMBER_REPORTS_PATH=${env.REAL_BASE_DIR}/outputs/${testRunnerIP}",
+                                "PLATFORM=${platform}",
+                                "SUITE=${suite}",
+                                "TAGS=${tags}",
+                        ]){
+                            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                                dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+                                sh(script: ".ci/scripts/generate-cucumber-reports.sh", label: "generate-cucumber-reports.sh")
+                            }
+                        }
+                        junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/${testRunnerIP}/TEST-*.xml")
+                        archiveArtifacts allowEmptyArchive: true,
+                                artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html"
+                    }
+                }
+            }
+        }
+    }
+}
+
+def retryWithNode(Map args = [:], Closure body) {
+    try {
+        incrementRetries(args.stageName)
+        withNode(args){
+            body()
+        }
+    } catch (err) {
+        log(level: 'WARN', text: "Stage '${args.stageName}' failed, let's analyse if it's a flaky CI worker.")
+        if (isFlakyWorker(args.stageName) && isRetryAvailable(args.stageName)) {
+            log(level: 'INFO', text: "Rerun '${args.stageName}' in a new worker.")
+            retryWithNode(args) {
+                body()
+            }
+        } else {
+            error("Error '${err.toString()}'")
+        }
+    }
+}
+
+def isFlakyWorker(stageName) {
+    if (workersStatus.containsKey(stageName)) {
+        return !workersStatus.get(stageName).get('status', true)
+    }
+    return false
+}
+
+def isRetryAvailable(stageName) {
+    return workersStatus.get(stageName).get('retries', 2) < 2
+}
+
+def incrementRetries(stageName) {
+    if (workersStatus.containsKey(stageName)) {
+        def current = workersStatus[stageName].get('retries', 0)
+        workersStatus[stageName].retries = current + 1
+    } else {
+        setFlakyWorker(stageName)
+        workersStatus[stageName].retries = 1
+    }
+}
+
+def setFlakyWorker(stageName) {
+    if (workersStatus.containsKey(stageName)) {
+        workersStatus[stageName].status = false
+    } else {
+        workersStatus[stageName] = [ status: false ]
+    }
+}
+
+def unsetFlakyWorker(stageName) {
+    workersStatus[stageName].status = true
+}
+
+def destroyStack(Map args = [:]) {
+    def stackMachine = getMachineInfo('stack')
+    if (!args.destroyCloudResources) {
+        def stackRunnerIP = getNodeIp('stack')
+        log(level: 'DEBUG', text: "Stack instance won't be destroyed after the build. Please SSH into the stack machine on ${stackRunnerIP}")
+    } else {
+        dir("${env.REAL_BASE_DIR}") {
+            ciBuild() {
+                retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                    sh(label: 'Destroy stack node', script: "make -C .ci destroy-stack")
+                }
+            }
+        }
+    }
+}
+
+return this
\ No newline at end of file
diff --git a/.ci/packer/aws-runners-windows.pkr.hcl b/.ci/packer/aws-runners-windows.pkr.hcl
deleted file mode 100644
index c2dfb014e8..0000000000
--- a/.ci/packer/aws-runners-windows.pkr.hcl
+++ /dev/null
@@ -1,55 +0,0 @@
-packer {
-  required_plugins {
-    amazon = {
-      version = ">= 1.1.5"
-      source  = "github.com/hashicorp/amazon"
-    }
-  }
-}
-
-variable "skip_create_ami" {
-  type = bool
-  default = false
-}
-
-locals {
-  aws_region = "us-east-2"
-  force_deregister = true
-}
-
-source "amazon-ebs" "windows2019" {
-  ami_name        = "windows-2019-runner-1"
-  instance_type   = "c5.2xlarge"
-  region          = local.aws_region
-  source_ami      = "ami-0587bd602f1da2f1d"
-  ssh_username  = "ogc"
-  communicator  = "ssh"
-  launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 60
-    volume_type = "gp3"
-  }
-
-  tags            = {
-    OS_Version  = "Windows"
-    Release     = "2019"
-    Arch        = "AMD64"
-  }
-  skip_create_ami  = var.skip_create_ami
-  force_deregister = local.force_deregister
-}
-
-build {
-  name = "e2e runners AMIs"
-  sources = [
-    "source.amazon-ebs.windows2019"
-  ]
-
-  provisioner "ansible" {
-    user = build.User
-    ansible_env_vars  =  ["PACKER_BUILD_NAME={{ build_name }}"]
-    playbook_file     = ".ci/ansible/playbook.yml"
-    extra_arguments   = ["--tags", "setup-ami", "--extra-vars", "nodeShellType=cmd"]
-    galaxy_file       = ".ci/ansible/requirements.yml"
-  }
-}
\ No newline at end of file
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 07dacf3e17..19307ee75c 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -8,157 +8,201 @@ packer {
 }
 
 variable "skip_create_ami" {
-  type = bool
+  type    = bool
   default = false
 }
 
+variable "source_set" {
+  type    = string
+  default = "linux"
+}
+
+variable "ami_suffix" {
+  type    = string
+  default = "test_suffix"
+}
+
 locals {
-  aws_region = "us-east-2"
+  aws_region       = "us-east-2"
   force_deregister = true
+  source_sets = {
+    "linux" = [
+      "source.amazon-ebs.ubuntu",
+      "source.amazon-ebs.debian-10-amd64",
+      "source.amazon-ebs.debian-10-arm64",
+      "source.amazon-ebs.debian-11-amd64",
+      "source.amazon-ebs.centos-8-amd64",
+      "source.amazon-ebs.centos-8-arm64",
+      "source.amazon-ebs.oracle-linux-8",
+      "source.amazon-ebs.sles15"
+    ]
+    "test" = ["source.amazon-ebs.ubuntu"]
+  }
 }
 
 source "amazon-ebs" "ubuntu" {
-  ami_name      = "ubuntu-2204-e2e-runner-1"
+  ami_name      = "ubuntu-2204-e2e-runner-${var.ami_suffix}"
   instance_type = "t3.xlarge"
   region        = local.aws_region
   source_ami    = "ami-0aeb7c931a5a61206"
   ssh_username  = "ubuntu"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true
   }
   tags = {
     OS_Version = "Ubuntu"
     Release    = "22.04"
     Arch       = "AMD64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
-  skip_create_ami = var.skip_create_ami
+  skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
 source "amazon-ebs" "debian-10-amd64" {
-  ami_name      = "debian-10-amd64-runner-1"
+  ami_name      = "debian-10-amd64-runner-${var.ami_suffix}"
   instance_type = "t3.xlarge"
   region        = local.aws_region
   source_ami    = "ami-0d90bed76900e679a"
   ssh_username  = "admin"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true
   }
   tags = {
     OS_Version = "Debian"
     Release    = "10"
     Arch       = "AMD64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
-  skip_create_ami = var.skip_create_ami
+  skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
 source "amazon-ebs" "debian-10-arm64" {
-  ami_name      = "debian-10-arm64-runner-1"
+  ami_name      = "debian-10-arm64-runner-${var.ami_suffix}"
   instance_type = "a1.large"
   region        = local.aws_region
   source_ami    = "ami-06dac44ad759182bd"
   ssh_username  = "admin"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true    
   }
   tags = {
     OS_Version = "Debian"
     Release    = "10"
     Arch       = "ARM64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
-  skip_create_ami = var.skip_create_ami
+  skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
 source "amazon-ebs" "debian-11-amd64" {
-  ami_name      = "debian-11-amd64-runner-1"
+  ami_name      = "debian-11-amd64-runner-${var.ami_suffix}"
   instance_type = "t3.xlarge"
   region        = local.aws_region
   source_ami    = "ami-0c7c4e3c6b4941f0f"
   ssh_username  = "admin"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true
   }
-  tags          = {
+  tags = {
     OS_Version = "Debian"
     Release    = "11"
     Arch       = "AMD64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
 source "amazon-ebs" "centos-8-amd64" {
-  ami_name      = "centos-8-amd64-runner-1"
+  ami_name      = "centos-8-amd64-runner-${var.ami_suffix}"
   instance_type = "t3.xlarge"
   region        = local.aws_region
   source_ami    = "ami-045b0a05944af45c1"
   ssh_username  = "centos"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true
   }
-  tags          = {
+  tags = {
     OS_Version = "Centos"
     Release    = "8"
     Arch       = "AMD64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
 source "amazon-ebs" "centos-8-arm64" {
-  ami_name      = "centos-8-arm64-runner-1"
+  ami_name      = "centos-8-arm64-runner-${var.ami_suffix}"
   instance_type = "a1.large"
   region        = local.aws_region
   source_ami    = "ami-01cdc9e8306344fe0"
   ssh_username  = "centos"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true
   }
-  tags          = {
+  tags = {
     OS_Version = "Centos"
     Release    = "8"
     Arch       = "ARM64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
 source "amazon-ebs" "oracle-linux-8" {
-  ami_name      = "oracle-linux-8-x86-64-runner-1"
+  ami_name      = "oracle-linux-8-x86-64-runner-${var.ami_suffix}"
   instance_type = "t3.xlarge"
   region        = local.aws_region
   source_ami    = "ami-00371eeb8fd8e0e16"
   ssh_username  = "ec2-user"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true
   }
-  tags          = {
+  tags = {
     OS_Version = "Oracle Linux"
     Release    = "8"
     Arch       = "x86-64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
@@ -166,44 +210,79 @@ source "amazon-ebs" "oracle-linux-8" {
 }
 
 source "amazon-ebs" "sles15" {
-  ami_name      = "sles15-runner-1"
+  ami_name      = "sles15-runner-${var.ami_suffix}"
   instance_type = "t3.xlarge"
   region        = local.aws_region
   source_ami    = "ami-0f7cb53c916a75006"
   ssh_username  = "ec2-user"
   communicator  = "ssh"
   launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 15
-    volume_type = "gp3"
+    device_name           = "/dev/sda1"
+    volume_size           = 15
+    volume_type           = "gp3"
+    delete_on_termination = true
   }
-  tags          = {
+  tags = {
     OS_Version = "SUSE Linux Enterprise Server 15 SP3"
     Release    = "8"
     Arch       = "ARM64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
+  }
+  skip_create_ami  = var.skip_create_ami
+  force_deregister = local.force_deregister
+}
+
+build {
+  name = "linux"
+
+  sources = local.source_sets[var.source_set]
+
+  provisioner "ansible" {
+    user             = build.User
+    ansible_env_vars = ["PACKER_BUILD_NAME={{ build_name }}"]
+    playbook_file    = "ansible/playbook.yml"
+    extra_arguments  = ["--tags", "setup-ami"]
+    galaxy_file      = "ansible/requirements.yml"
+  }
+}
+
+# Windows
+source "amazon-ebs" "windows2019" {
+  ami_name      = "windows-2019-runner-${var.ami_suffix}"
+  instance_type = "c5.2xlarge"
+  region        = local.aws_region
+  source_ami    = "ami-0587bd602f1da2f1d"
+  ssh_username  = "ogc"
+  communicator  = "ssh"
+  launch_block_device_mappings {
+    device_name           = "/dev/sda1"
+    volume_size           = 60
+    volume_type           = "gp3"
+    delete_on_termination = true
+  }
+  tags = {
+    OS_Version = "Windows"
+    Release    = "2019"
+    Arch       = "x86_64"
+    Branch     = var.ami_suffix
+    Project    = "e2e"
   }
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
 
 build {
-  name = "e2e runners AMIs"
+  name = "windows"
   sources = [
-    "source.amazon-ebs.ubuntu",
-    "source.amazon-ebs.debian-10-amd64",
-    "source.amazon-ebs.debian-10-arm64",
-    "source.amazon-ebs.debian-11-amd64",
-    "source.amazon-ebs.centos-8-amd64",
-    "source.amazon-ebs.centos-8-arm64",
-    "source.amazon-ebs.oracle-linux-8",
-    "source.amazon-ebs.sles15"
+    "source.amazon-ebs.windows2019"
   ]
 
   provisioner "ansible" {
-    user = build.User
-    ansible_env_vars  =  ["PACKER_BUILD_NAME={{ build_name }}"]
-    playbook_file     = ".ci/ansible/playbook.yml"
-    extra_arguments   = ["--tags", "setup-ami"]
-    galaxy_file       = ".ci/ansible/requirements.yml"
+    user             = build.User
+    ansible_env_vars = ["PACKER_BUILD_NAME={{ build_name }}"]
+    playbook_file    = "ansible/playbook.yml"
+    extra_arguments  = ["--tags", "setup-ami", "--extra-vars", "nodeShellType=cmd"]
+    galaxy_file      = "ansible/requirements.yml"
   }
 }
\ No newline at end of file

From 7209c3d8115ed58dca541fcdcfb44ec095a2eae0 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 5 Dec 2022 17:25:01 +0100
Subject: [PATCH 25/78] Fixed missin comma

---
 .ci/Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 20a9cf5ad4..251a8fe444 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -198,13 +198,13 @@ pipeline {
           steps {
             withGithubNotify(context: 'E2E Tests', tab: 'tests') {
               script {                
-                def existingSuites = readYaml(file: "${env.REAL_BASE_DIR}/${TEST_MATRIX_FILE}")
+                def testMatrix = readYaml(file: "${env.REAL_BASE_DIR}/${TEST_MATRIX_FILE}")
                 runE2ETests(
-                  testMatrix: existingSuites,
+                  testMatrix: testMatrix,
                   selectedSuites: env.RUN_TEST_SUITES,
                   runAsMainBranch: env.RUN_AS_MAIN_BRANCH,
                   githubCheckSha1: env.GITHUB_CHECK_SHA1,
-                  githubCheckRepo: env.GITHUB_CHECK_REP
+                  githubCheckRepo: env.GITHUB_CHECK_REP,
                   destroyTestRunner: env.DESTROY_CLOUD_RESOURCES
                 )
               }

From d6de2d979bd5b4f614c2a5ae5b997d24ea29be4a Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 5 Dec 2022 17:33:30 +0100
Subject: [PATCH 26/78] Added required script blocks

---
 .ci/Jenkinsfile | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 251a8fe444..e274f510c8 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -106,9 +106,11 @@ pipeline {
             stash allowEmpty: true, name: 'source', useDefaultExcludes: false
             setEnvVar("GO_VERSION", readFile("${env.REAL_BASE_DIR}/.go-version").trim())
             setEnvVar("LABELS_STRING", "buildURL=${env.BUILD_URL} gitSha=${env.GIT_BASE_COMMIT} build=${env.BUILD_ID} repo=${env.REPO} branch=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')} type=ci")
-            def e2eUtils = load ".ci/e2eUtils.groovy"
-            e2eUtils.checkSkipTests()
-            e2eUtils.githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
+            script {
+              def e2eUtils = load ".ci/e2eUtils.groovy"            
+              e2eUtils.checkSkipTests()
+              e2eUtils.githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
+            }
           }
         }
         stage('Build Docs') {
@@ -243,11 +245,13 @@ pipeline {
   }
   post {
     cleanup {
-      e2eUtils.doNotifyBuildResult(
-        slackNotify: params.notifyOnGreenBuilds,
-        slackChannel: env.SLACK_CHANNEL,
-        runTestsSuites: env.RUN_TEST_SUITES
-      )
+      script {
+        e2eUtils.doNotifyBuildResult(
+          slackNotify: params.notifyOnGreenBuilds,
+          slackChannel: env.SLACK_CHANNEL,
+          runTestsSuites: env.RUN_TEST_SUITES
+        )
+      }
     }
   }
 }

From ab4df075cb181fb9281732e393522691e4d3aca3 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 5 Dec 2022 21:34:45 +0100
Subject: [PATCH 27/78] Fixed utils path

---
 .ci/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index e274f510c8..3a0a4c992b 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -107,7 +107,7 @@ pipeline {
             setEnvVar("GO_VERSION", readFile("${env.REAL_BASE_DIR}/.go-version").trim())
             setEnvVar("LABELS_STRING", "buildURL=${env.BUILD_URL} gitSha=${env.GIT_BASE_COMMIT} build=${env.BUILD_ID} repo=${env.REPO} branch=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')} type=ci")
             script {
-              def e2eUtils = load ".ci/e2eUtils.groovy"            
+              def e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"            
               e2eUtils.checkSkipTests()
               e2eUtils.githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
             }

From 39b7231b213dfb3c0969b24895f56b479d59c5c9 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 5 Dec 2022 22:50:50 +0100
Subject: [PATCH 28/78] Declared e2eUtils globally

---
 .ci/Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 3a0a4c992b..452da33c81 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -9,6 +9,7 @@ Store the worker status so if the CI worker behaves wrongy then let's rerun the
 */
 @Field def workersStatus = [:]
 
+def e2eUtils
 pipeline {
   agent { label 'ubuntu-20.04 && immutable && docker' }
   environment {
@@ -107,7 +108,7 @@ pipeline {
             setEnvVar("GO_VERSION", readFile("${env.REAL_BASE_DIR}/.go-version").trim())
             setEnvVar("LABELS_STRING", "buildURL=${env.BUILD_URL} gitSha=${env.GIT_BASE_COMMIT} build=${env.BUILD_ID} repo=${env.REPO} branch=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')} type=ci")
             script {
-              def e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"            
+              e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"            
               e2eUtils.checkSkipTests()
               e2eUtils.githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
             }

From 257b866d5400af20838be47eeae6f0aa90d81023 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 5 Dec 2022 23:00:36 +0100
Subject: [PATCH 29/78] Declared e2eUtils globally

---
 .ci/Jenkinsfile     | 1 +
 .ci/e2eUtils.groovy | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 452da33c81..65e0e6d356 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -10,6 +10,7 @@ Store the worker status so if the CI worker behaves wrongy then let's rerun the
 @Field def workersStatus = [:]
 
 def e2eUtils
+
 pipeline {
   agent { label 'ubuntu-20.04 && immutable && docker' }
   environment {
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index b7c7cd37ef..aa5a79fee6 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -162,7 +162,7 @@ def doNotifyBuildResult(Map args = [:]) {
         doSlackNotify = args.slackNotify // if the build status is success, read the parameter
     }
 
-    githubCheckNotify(githubCheckStatus)
+    this.githubCheckNotify(githubCheckStatus)
 
     def testsSuites = args.runTestsSuites?.trim() ?: "All suites"    
     def channels = args.slackChannel?.trim() ?: "observablt-bots"    

From ab11d1f34adccea8817c485850556e16cf091958 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 5 Dec 2022 23:11:44 +0100
Subject: [PATCH 30/78] Fixed githubCheckNotify declaration

---
 .ci/e2eUtils.groovy | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index aa5a79fee6..f879fd4b2e 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -103,10 +103,16 @@ def checkSkipTests() {
         }
 
         // patterns for all places that should trigger a full build
-        def tests_regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/helm/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
+        def tests_regexps = [ 
+            "^e2e/_suites/fleet/.*", 
+            "^e2e/_suites/helm/.*", 
+            "^e2e/_suites/kubernetes-autodiscover/.*", 
+            "^.ci/.*", 
+            "^cli/.*", 
+            "^e2e/.*\\.go", 
+            "^internal/.*\\.go" ]
         // def ami_regexps = [ "^.ci/ansible/.*", "^.ci/packer/.*"]
-        setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: tests_regexps, shouldMatchAll: false))
-        
+        setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: tests_regexps, shouldMatchAll: false))        
     }
 }
 
@@ -180,7 +186,7 @@ def doNotifyBuildResult(Map args = [:]) {
 /**
  Notify the GitHub check of the parent stream
  **/
-def githubCheckNotify(Map args = [:] String status, String githubCheckName, String githubCheckRepo, String githubCheckSha1) {
+def githubCheckNotify(Map args = [:]) {
     if (args.githubCheckName?.trim() && args.githubCheckRepo?.trim() && args.githubCheckSha1?.trim()) {
         githubNotify context: "${args.githubCheckName}",
                 description: "${args.githubCheckName} ${args.status?.toLowerCase()}",

From 4b8f6242dde39fae7b03f8d1743f853b8980c4cf Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 00:00:49 +0100
Subject: [PATCH 31/78] Fixed githubCheckNotify declaration

---
 .ci/Jenkinsfile     | 13 +++++++++++--
 .ci/e2eUtils.groovy |  7 ++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 65e0e6d356..d3a933c7b4 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -111,7 +111,13 @@ pipeline {
             script {
               e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"            
               e2eUtils.checkSkipTests()
-              e2eUtils.githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
+              echo '<<OK>>'
+              e2eUtils.githubCheckNotify(
+                status: 'PENDING',
+                githubCheckRepo: env.GITHUB_CHECK_REPO,
+                githubCheckSha1: env.GITHUB_CHECK_SHA1,
+                githubCheckName: env.GITHUB_GITHUB_CHECK_NAME
+              )
             }
           }
         }
@@ -251,7 +257,10 @@ pipeline {
         e2eUtils.doNotifyBuildResult(
           slackNotify: params.notifyOnGreenBuilds,
           slackChannel: env.SLACK_CHANNEL,
-          runTestsSuites: env.RUN_TEST_SUITES
+          runTestsSuites: env.RUN_TEST_SUITES,
+          githubCheckRepo: env.GITHUB_CHECK_REPO,
+          githubCheckSha1: env.GITHUB_CHECK_SHA1,
+          githubCheckName: env.GITHUB_GITHUB_CHECK_NAME
         )
       }
     }
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index f879fd4b2e..6294456849 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -168,7 +168,12 @@ def doNotifyBuildResult(Map args = [:]) {
         doSlackNotify = args.slackNotify // if the build status is success, read the parameter
     }
 
-    this.githubCheckNotify(githubCheckStatus)
+    this.githubCheckNotify(
+        status: githubCheckStatus,
+        githubCheckRepo: args.githubCheckRepo,
+        githubCheckSha1: args.githubCheckSha1,
+        githubCheckName: args.githubCheckName
+    )
 
     def testsSuites = args.runTestsSuites?.trim() ?: "All suites"    
     def channels = args.slackChannel?.trim() ?: "observablt-bots"    

From 3254209755655720f4e6ada850ce16bb8d8eb676 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 10:41:45 +0100
Subject: [PATCH 32/78] Fixed rune2e declaration

---
 .ci/Jenkinsfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index d3a933c7b4..99501cd568 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -170,16 +170,16 @@ pipeline {
 
                 dir("${env.REAL_BASE_DIR}") {
                   e2eUtils.ciBuild() {
-                    sh(label: 'Create Stack node', script: "make -C .ci provision-stack")
+                    // sh(label: 'Create Stack node', script: "make -C .ci provision-stack")
                   }
                   e2eUtils.ciBuild() {
                     retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                      sh(label: 'Setup Stack node', script: "make -C .ci setup-stack")
+                      // sh(label: 'Setup Stack node', script: "make -C .ci setup-stack")
                     }
                   }
                   e2eUtils.ciBuild() {
                     retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                      sh(label: 'Start Elastic Stack', script: "make -C .ci start-elastic-stack")
+                      // sh(label: 'Start Elastic Stack', script: "make -C .ci start-elastic-stack")
                     }
                   }
 
@@ -209,7 +209,7 @@ pipeline {
             withGithubNotify(context: 'E2E Tests', tab: 'tests') {
               script {                
                 def testMatrix = readYaml(file: "${env.REAL_BASE_DIR}/${TEST_MATRIX_FILE}")
-                runE2ETests(
+                e2eUtils.runE2ETests(
                   testMatrix: testMatrix,
                   selectedSuites: env.RUN_TEST_SUITES,
                   runAsMainBranch: env.RUN_AS_MAIN_BRANCH,
@@ -224,7 +224,7 @@ pipeline {
             cleanup {
               // Once all tests are complete we need to teardown the single instance with the deployed stack
               script {
-                destroyStack(destroyCloudResources: env.DESTROY_CLOUD_RESOURCES)
+                e2eUtils.destroyStack(destroyCloudResources: env.DESTROY_CLOUD_RESOURCES)
               }
             }
           }

From 5619efc6cf30344ed88cbf1ab1ff541e212def95 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 11:17:11 +0100
Subject: [PATCH 33/78] Fixed rune2e suites condition

---
 .ci/e2eUtils.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index 6294456849..3cebaa551c 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -1,6 +1,6 @@
 def runE2ETests(Map args = [:]) {
     def parallelTasks = [:]
-    if (args.selectedSuites?.trim(0)) {
+    if (!args.selectedSuites?.trim()) {
         log(level: 'DEBUG', text: "Iterate through existing test suites")
         args.testMatrix['SUITES'].each { item ->
             parallelTasks += convertSuiteToTasks(
@@ -14,7 +14,7 @@ def runE2ETests(Map args = [:]) {
         }
     } else {
         log(level: 'DEBUG', text: "Iterate through the comma-separated test suites (${args.selectedSuites}), comparing with the existing test suites")
-        args.selectedSuites.split(',').each { selectedSuite ->
+        args.selectedSuites?.split(',')?.each { selectedSuite ->
             args.testMatrix['SUITES'].findAll { selectedSuite.trim() == it.suite }.each { item ->
                 parallelTasks += convertSuiteToTasks(
                     item: item,

From b7499605d4541e8853ab0c68ef189d10956a6c55 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 11:28:22 +0100
Subject: [PATCH 34/78] Init e2eUtils with workerStatus

---
 .ci/Jenkinsfile     | 3 ++-
 .ci/e2eUtils.groovy | 8 ++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 99501cd568..d10a6f5e59 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -109,7 +109,8 @@ pipeline {
             setEnvVar("GO_VERSION", readFile("${env.REAL_BASE_DIR}/.go-version").trim())
             setEnvVar("LABELS_STRING", "buildURL=${env.BUILD_URL} gitSha=${env.GIT_BASE_COMMIT} build=${env.BUILD_ID} repo=${env.REPO} branch=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')} type=ci")
             script {
-              e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"            
+              e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"
+              e2eUtils.init(workersStatus)
               e2eUtils.checkSkipTests()
               echo '<<OK>>'
               e2eUtils.githubCheckNotify(
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index 3cebaa551c..ddc6e8e51c 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -1,3 +1,11 @@
+import groovy.transform.Field
+
+@Field private workersStatus = [:]
+
+def init(workersStatus) {
+    this.workersStatus = workersStatus
+}
+
 def runE2ETests(Map args = [:]) {
     def parallelTasks = [:]
     if (!args.selectedSuites?.trim()) {

From d277a5aa89192251cb9fdc600adfab1e8f24358a Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 23:22:22 +0100
Subject: [PATCH 35/78] troubleshooting nodeImage

---
 .ci/.e2e-tests-tmp.yaml |  2 +-
 .ci/Jenkinsfile         | 15 +++------------
 .ci/e2eUtils.groovy     | 17 ++++++++---------
 3 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/.ci/.e2e-tests-tmp.yaml b/.ci/.e2e-tests-tmp.yaml
index 2c48b37547..5f71503292 100644
--- a/.ci/.e2e-tests-tmp.yaml
+++ b/.ci/.e2e-tests-tmp.yaml
@@ -16,4 +16,4 @@ SUITES:
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["windows2019_preinstalled", "centos8_amd64_preinstalled"]
+        platforms: ["centos8_amd64_preinstalled"]
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index d10a6f5e59..7d074ef056 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -2,14 +2,7 @@
 
 @Library('apm@current') _
 
-import groovy.transform.Field
-
-/**
-Store the worker status so if the CI worker behaves wrongy then let's rerun the stage again
-*/
-@Field def workersStatus = [:]
-
-def e2eUtils
+@Field def e2eUtils
 
 pipeline {
   agent { label 'ubuntu-20.04 && immutable && docker' }
@@ -109,10 +102,8 @@ pipeline {
             setEnvVar("GO_VERSION", readFile("${env.REAL_BASE_DIR}/.go-version").trim())
             setEnvVar("LABELS_STRING", "buildURL=${env.BUILD_URL} gitSha=${env.GIT_BASE_COMMIT} build=${env.BUILD_ID} repo=${env.REPO} branch=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')} type=ci")
             script {
-              e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"
-              e2eUtils.init(workersStatus)
-              e2eUtils.checkSkipTests()
-              echo '<<OK>>'
+              e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"              
+              e2eUtils.checkSkipTests()              
               e2eUtils.githubCheckNotify(
                 status: 'PENDING',
                 githubCheckRepo: env.GITHUB_CHECK_REPO,
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index ddc6e8e51c..dd8a928792 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -1,10 +1,9 @@
 import groovy.transform.Field
 
-@Field private workersStatus = [:]
-
-def init(workersStatus) {
-    this.workersStatus = workersStatus
-}
+/**
+Store the worker status so if the CI worker behaves wrongy then let's rerun the stage again
+*/
+@Field def workersStatus = [:]
 
 def runE2ETests(Map args = [:]) {
     def parallelTasks = [:]
@@ -224,9 +223,8 @@ def generateFunctionalTestStep(Map args = [:]) {
     def tags = args.get('tags')
     def pullRequestFilter = args.get('pullRequestFilter')?.trim() ?: ''
     def machine = args.get('machine')
-    def stageName = args.get('stageName')
-    //TODO
-    def ami_suffix = args.amiSuffix.trim() ?: 'main'
+    def stageName = args.get('stageName')    
+    def amiSuffix = args.amiSuffix.trim() ?: 'main'
     def runAsMainBranch = args.runAsMainBranch ?: false
     def destroyTestRunner = args.destroyTestRunner ?: false
     
@@ -261,7 +259,7 @@ def generateFunctionalTestStep(Map args = [:]) {
     envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},elastic_agent_version=${env.ELASTIC_AGENT_VERSION},stack_version=${env.STACK_VERSION}")
     // VM characteristics
     envContext.add("NODE_LABEL=${platform}")
-    envContext.add("NODE_IMAGE=${machine.image}-${ami_suffix}")
+    envContext.add("NODE_IMAGE=${machine.image}-${amiSuffix}")
     envContext.add("NODE_INSTANCE_ID=${env.BUILD_URL}_${platform}_${suite}_${tags}")
     envContext.add("NODE_INSTANCE_TYPE=${machine.instance_type}")
     envContext.add("NODE_SHELL_TYPE=${machine.shell_type}")
@@ -282,6 +280,7 @@ def generateFunctionalTestStep(Map args = [:]) {
                         // IMPORTANT: withAPMEnv is now the one in used since withOtelEnv uses a specific Opentelemetry Collector at the moment.
                         // TODO: This will need to be integrated into the provisioned VMs
                         withAPMEnv() {
+                            echo "nodeImage: ${env.NODE_IMAGE}"
                             // we are separating the different test phases to avoid recreating
                             ciBuild() {
                                 sh(label: 'Start node', script: "make -C .ci provision-node")

From 4941c8f651c1e44d76c8e8c3fd198f715b6477ca Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 23:27:08 +0100
Subject: [PATCH 36/78] Fix imports

---
 .ci/Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 7d074ef056..8bea3bcc3b 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -1,4 +1,5 @@
 #!/usr/bin/env groovy
+import groovy.transform.Field
 
 @Library('apm@current') _
 

From 432476a0d509eaea76931f34fa35f6d36644eb7e Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 23:46:32 +0100
Subject: [PATCH 37/78] Fix imports

---
 .ci/ansible/tasks/runners.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 95e272ca9f..e0f6af26df 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -31,6 +31,10 @@
   tags:    
     - provision-node
     - provision-stack
+ 
+  - name: Print found images
+    ansible.builtin.debug:
+      var: preinstalled_images   
 
 - name: "Create preinstalled {{nodeLabel}} AWS instances"
   vars:

From d982fbe905148bfc110f8404345b70656448a0ae Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 6 Dec 2022 23:47:29 +0100
Subject: [PATCH 38/78] Fix imports

---
 .ci/ansible/tasks/runners.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index e0f6af26df..9ed2c00712 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -34,7 +34,10 @@
  
   - name: Print found images
     ansible.builtin.debug:
-      var: preinstalled_images   
+      var: preinstalled_images
+    tags:
+      - provision-node
+      - provision-stack  
 
 - name: "Create preinstalled {{nodeLabel}} AWS instances"
   vars:

From 7dac93625b54455d39230bf3896fb1622ecdf35c Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 00:24:42 +0100
Subject: [PATCH 39/78] Fix imports

---
 .ci/ansible/tasks/runners.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 9ed2c00712..3e5f956fdb 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -32,12 +32,12 @@
     - provision-node
     - provision-stack
  
-  - name: Print found images
-    ansible.builtin.debug:
-      var: preinstalled_images
-    tags:
-      - provision-node
-      - provision-stack  
+- name: Print found images
+  ansible.builtin.debug:
+    var: preinstalled_images
+  tags:
+    - provision-node
+    - provision-stack  
 
 - name: "Create preinstalled {{nodeLabel}} AWS instances"
   vars:

From 5e3c35e9febdbd5bea7b31fa90e4683d24a988d1 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 12:43:55 +0100
Subject: [PATCH 40/78] Added organisation arn variable

---
 .ci/Makefile                   |  7 +++++--
 .ci/packer/aws-runners.pkr.hcl | 27 ++++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index d6fe8900a0..ecb1bcdb1e 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -45,6 +45,9 @@ TAGS ?= fleet_mode
 # used in .ci/packer/aws-runners.pkr.hcl
 AMI_SUFFIX ?= main
 
+# AWS organisation arn. Specifies organisation that packer will grant access to the AMIs
+AWS_ORG_ARN ?= 
+
 # Check that given variables are set and all have non-empty values,
 # die with an error otherwise.
 #
@@ -252,5 +255,5 @@ validate-packer-template:
 
 .PHONY: validate-packer-template
 build-amis-$(AMI_SUFFIX): validate-packer-template
-	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)	
-	packer build -var "ami_suffix=${AMI_SUFFIX}" $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
+	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)		
+	packer build -var "ami_suffix=$(AMI_SUFFIX)" -var "org_arn=$(AWS_ORG_ARN)" $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 19307ee75c..82302f5ee2 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -17,6 +17,11 @@ variable "source_set" {
   default = "linux"
 }
 
+variable "org_arn" {
+  type  = string
+  default = ""
+}
+
 variable "ami_suffix" {
   type    = string
   default = "test_suffix"
@@ -35,8 +40,20 @@ locals {
       "source.amazon-ebs.centos-8-arm64",
       "source.amazon-ebs.oracle-linux-8",
       "source.amazon-ebs.sles15"
+    ],
+    "test" = ["source.amazon-ebs.ubuntu"],
+    "windows" = ["source.amazon-ebs.windows2019"],
+    "all" = [
+      "source.amazon-ebs.ubuntu",
+      "source.amazon-ebs.debian-10-amd64",
+      "source.amazon-ebs.debian-10-arm64",
+      "source.amazon-ebs.debian-11-amd64",
+      "source.amazon-ebs.centos-8-amd64",
+      "source.amazon-ebs.centos-8-arm64",
+      "source.amazon-ebs.oracle-linux-8",
+      "source.amazon-ebs.sles15",
+      "source.amazon-ebs.windows2019"
     ]
-    "test" = ["source.amazon-ebs.ubuntu"]
   }
 }
 
@@ -60,6 +77,7 @@ source "amazon-ebs" "ubuntu" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -84,6 +102,7 @@ source "amazon-ebs" "debian-10-amd64" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -108,6 +127,7 @@ source "amazon-ebs" "debian-10-arm64" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -132,6 +152,7 @@ source "amazon-ebs" "debian-11-amd64" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -156,6 +177,7 @@ source "amazon-ebs" "centos-8-amd64" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -180,6 +202,7 @@ source "amazon-ebs" "centos-8-arm64" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -204,6 +227,7 @@ source "amazon-ebs" "oracle-linux-8" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 
@@ -229,6 +253,7 @@ source "amazon-ebs" "sles15" {
     Branch     = var.ami_suffix
     Project    = "e2e"
   }
+  ami_org_arns = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }

From fedb5e7589fd8a420e6664143ffd6bddcf290d65 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 13:38:37 +0100
Subject: [PATCH 41/78] Added packer build step

---
 .ci/Jenkinsfile     | 19 ++++++++++++++++++-
 .ci/Makefile        |  6 +++++-
 .ci/e2eUtils.groovy | 10 +++++++++-
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 8bea3bcc3b..6a43bb9d07 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -104,7 +104,8 @@ pipeline {
             setEnvVar("LABELS_STRING", "buildURL=${env.BUILD_URL} gitSha=${env.GIT_BASE_COMMIT} build=${env.BUILD_ID} repo=${env.REPO} branch=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')} type=ci")
             script {
               e2eUtils = load "${env.REAL_BASE_DIR}/.ci/e2eUtils.groovy"              
-              e2eUtils.checkSkipTests()              
+              e2eUtils.checkSkipTests()
+              e2eUtils.checkRebuildAmis()
               e2eUtils.githubCheckNotify(
                 status: 'PENDING',
                 githubCheckRepo: env.GITHUB_CHECK_REPO,
@@ -137,6 +138,22 @@ pipeline {
             }
           }
         }
+
+        stage("Build runner AMIs") {
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            anyOf {
+              expression { return env.REBUILD_AMIS == "true" }
+            }
+            steps {
+              script {
+                e2eUtils.buildPackerAMIs(amiSuffix: 'main')
+              }
+            }
+          }
+        }
+
         stage('Deploy Test Infra') {
           failFast true
           options { skipDefaultCheckout() }
diff --git a/.ci/Makefile b/.ci/Makefile
index ecb1bcdb1e..e64b402548 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -249,11 +249,15 @@ run-tests-win: set-env-$(NODE_LABEL)
 		-i $(NODE_IP_ADDRESS),
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $${NODE_USER}@$(NODE_IP_ADDRESS) "powershell \"C:/Users/$${NODE_USER}/e2e-testing/.ci/scripts/functional-test.ps1\""
 
+.PHONY: packer-init
+packer-init: 
+	packer init $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
+
 .PHONY: validate-packer-template
 validate-packer-template:	
 	packer validate $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
 
-.PHONY: validate-packer-template
+.PHONY: packer-init validate-packer-template 
 build-amis-$(AMI_SUFFIX): validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)		
 	packer build -var "ami_suffix=$(AMI_SUFFIX)" -var "org_arn=$(AWS_ORG_ARN)" $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index dd8a928792..fafb646cc0 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -126,7 +126,7 @@ def checkSkipTests() {
 /*
  * Runs the Make build at the CI, executing the closure in the context of Ansible + AWS
  */
-def ciBuild(Closure body){
+def ciBuild(Closure body) {
     withEnv([
             "SSH_KEY=${E2E_SSH_KEY}"
     ]) {
@@ -355,6 +355,14 @@ def generateFunctionalTestStep(Map args = [:]) {
     }
 }
 
+def buildPackerAMIs(Map args = [:]) {
+    ciBuild() {
+        
+    }
+    
+    
+}
+
 def retryWithNode(Map args = [:], Closure body) {
     try {
         incrementRetries(args.stageName)

From 159c1bb3a255fe9c30c6d39ffc8bb09c70d60c2a Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 13:54:56 +0100
Subject: [PATCH 42/78] Added packer build step

---
 .ci/Jenkinsfile | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 6a43bb9d07..1e360accf9 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -139,19 +139,23 @@ pipeline {
           }
         }
 
-        stage("Build runner AMIs") {
+        stage('Build runner AMIs') {
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
             anyOf {
               expression { return env.REBUILD_AMIS == "true" }
             }
-            steps {
+          }
+          steps {
+            withGithubNotify(context: 'Build runner AMIs', tab: 'tests') {
+              deleteDir()
+              unstash 'source'
               script {
                 e2eUtils.buildPackerAMIs(amiSuffix: 'main')
               }
             }
-          }
+          }          
         }
 
         stage('Deploy Test Infra') {
@@ -216,6 +220,8 @@ pipeline {
             }
           }
           steps {
+            deleteDir()
+            unstash 'source'
             withGithubNotify(context: 'E2E Tests', tab: 'tests') {
               script {                
                 def testMatrix = readYaml(file: "${env.REAL_BASE_DIR}/${TEST_MATRIX_FILE}")

From 60ff1ed4c46d62aff70ed86aedede0800fbf9066 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 13:59:23 +0100
Subject: [PATCH 43/78] Added packer build step

---
 .ci/e2eUtils.groovy | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index fafb646cc0..ccadece92a 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -93,9 +93,8 @@ def convertSuiteToTasks(Map args = [:]) {
 }
 
 def checkRebuildAmis() {
-    dir("${BASE_DIR}") {
-        def ami_regexps = [ "^.ci/ansible/.*", "^.ci/packer/.*"]
-        setEnvVar("REBUILD_AMIS", isGitRegionMatch(patterns: tests_regexps, shouldMatchAll: false))
+    dir("${BASE_DIR}") {        
+        setEnvVar("REBUILD_AMIS", isGitRegionMatch(patterns: [ "^.ci/ansible/.*", "^.ci/packer/.*"], shouldMatchAll: false))
     }
 }
 
@@ -356,11 +355,14 @@ def generateFunctionalTestStep(Map args = [:]) {
 }
 
 def buildPackerAMIs(Map args = [:]) {
+    
+    if (!args.amiSuffix?.trim()) {
+        error("amiSuffix parameter must be specified in buildPackerAMIs()")
+    }
+    setEnvVar("AMI_SUFFIX", args.amiSuffix)
     ciBuild() {
-        
+        sh(label: "Build AMIS with suffix:${args.amiSuffix}", script: 'make -C .ci build-amis-$AMI_SUFFIX')
     }
-    
-    
 }
 
 def retryWithNode(Map args = [:], Closure body) {

From 3939873fa580f31b4babfe4a7e627a6aa1fd0182 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 14:05:03 +0100
Subject: [PATCH 44/78] Added packer build step

---
 .ci/e2eUtils.groovy | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index ccadece92a..d14997eaed 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -93,7 +93,7 @@ def convertSuiteToTasks(Map args = [:]) {
 }
 
 def checkRebuildAmis() {
-    dir("${BASE_DIR}") {        
+    dir("${BASE_DIR}") {
         setEnvVar("REBUILD_AMIS", isGitRegionMatch(patterns: [ "^.ci/ansible/.*", "^.ci/packer/.*"], shouldMatchAll: false))
     }
 }
@@ -355,13 +355,14 @@ def generateFunctionalTestStep(Map args = [:]) {
 }
 
 def buildPackerAMIs(Map args = [:]) {
-    
-    if (!args.amiSuffix?.trim()) {
-        error("amiSuffix parameter must be specified in buildPackerAMIs()")
-    }
-    setEnvVar("AMI_SUFFIX", args.amiSuffix)
-    ciBuild() {
-        sh(label: "Build AMIS with suffix:${args.amiSuffix}", script: 'make -C .ci build-amis-$AMI_SUFFIX')
+    dir("${BASE_DIR}") {
+        if (!args.amiSuffix?.trim()) {
+            error("amiSuffix parameter must be specified in buildPackerAMIs()")
+        }
+        setEnvVar("AMI_SUFFIX", args.amiSuffix)
+        ciBuild() {
+            sh(label: "Build AMIS with suffix:${args.amiSuffix}", script: 'make -C .ci build-amis-$AMI_SUFFIX')
+        }
     }
 }
 

From bb7adbd9a5b9363f402680d871e1da91d140a3af Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 14:35:43 +0100
Subject: [PATCH 45/78] Added packer build step

---
 .ci/ansible/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/ansible/requirements.txt b/.ci/ansible/requirements.txt
index be0d348bcd..3047cb6c67 100644
--- a/.ci/ansible/requirements.txt
+++ b/.ci/ansible/requirements.txt
@@ -7,3 +7,4 @@ opentelemetry-exporter-otlp
 opentelemetry-sdk
 protobuf==3.20.1 #Temporary change because of protobuf new version bug: https://github.com/protocolbuffers/protobuf/issues/10051
 requests
+packer

From eb7d19061881e4ce40237c23f5a862c6017b4a23 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 15:13:18 +0100
Subject: [PATCH 46/78] added withPackerEnv

---
 .ci/ansible/requirements.txt | 1 -
 .ci/e2eUtils.groovy          | 4 +++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.ci/ansible/requirements.txt b/.ci/ansible/requirements.txt
index 3047cb6c67..be0d348bcd 100644
--- a/.ci/ansible/requirements.txt
+++ b/.ci/ansible/requirements.txt
@@ -7,4 +7,3 @@ opentelemetry-exporter-otlp
 opentelemetry-sdk
 protobuf==3.20.1 #Temporary change because of protobuf new version bug: https://github.com/protocolbuffers/protobuf/issues/10051
 requests
-packer
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index d14997eaed..7ebfdf18d0 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -361,7 +361,9 @@ def buildPackerAMIs(Map args = [:]) {
         }
         setEnvVar("AMI_SUFFIX", args.amiSuffix)
         ciBuild() {
-            sh(label: "Build AMIS with suffix:${args.amiSuffix}", script: 'make -C .ci build-amis-$AMI_SUFFIX')
+            withPackerEnv() {
+                sh(label: "Build AMIS with suffix:${args.amiSuffix}", script: 'make -C .ci build-amis-$AMI_SUFFIX')
+            }
         }
     }
 }

From 15e41b7f387ff93f0f2fb57f52bda99e018666a6 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 15:14:51 +0100
Subject: [PATCH 47/78] added withPackerEnv

---
 .ci/e2eUtils.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index 7ebfdf18d0..fac720dbdc 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -361,7 +361,7 @@ def buildPackerAMIs(Map args = [:]) {
         }
         setEnvVar("AMI_SUFFIX", args.amiSuffix)
         ciBuild() {
-            withPackerEnv() {
+            withPackerEnv(version: '1.8.1') {
                 sh(label: "Build AMIS with suffix:${args.amiSuffix}", script: 'make -C .ci build-amis-$AMI_SUFFIX')
             }
         }

From 4e9d4dab651126a6060e5c2d09a45756878cfa58 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 17:59:58 +0100
Subject: [PATCH 48/78] added withPackerEnv

---
 .ci/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index e64b402548..f8efcdba87 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -257,7 +257,7 @@ packer-init:
 validate-packer-template:	
 	packer validate $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
 
-.PHONY: packer-init validate-packer-template 
-build-amis-$(AMI_SUFFIX): validate-packer-template
+.PHONY: validate-packer-template 
+build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)		
 	packer build -var "ami_suffix=$(AMI_SUFFIX)" -var "org_arn=$(AWS_ORG_ARN)" $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl

From 8c69a6a851eb169fd13dc871d852cb6efa77c8e7 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 7 Dec 2022 19:22:08 +0100
Subject: [PATCH 49/78] added withPackerEnv

---
 .ci/ansible/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/ansible/requirements.txt b/.ci/ansible/requirements.txt
index be0d348bcd..eb85382ead 100644
--- a/.ci/ansible/requirements.txt
+++ b/.ci/ansible/requirements.txt
@@ -1,5 +1,6 @@
 ###### Requirements without Version Specifiers ######
 ansible
+ansible-playbook
 boto
 boto3
 opentelemetry-api

From 84d6b4b9a7347eb172263b7b27775f6d4e0380b6 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 8 Dec 2022 17:00:23 +0100
Subject: [PATCH 50/78] Specified ansible paths

---
 .ci/Makefile                   |  7 ++++++-
 .ci/ansible/requirements.txt   |  1 -
 .ci/packer/aws-runners.pkr.hcl | 12 ++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index f8efcdba87..997ff833a8 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -260,4 +260,9 @@ validate-packer-template:
 .PHONY: validate-packer-template 
 build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)		
-	packer build -var "ami_suffix=$(AMI_SUFFIX)" -var "org_arn=$(AWS_ORG_ARN)" $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
+	packer build \
+		-var "ami_suffix=$(AMI_SUFFIX)" \
+		-var "org_arn=$(AWS_ORG_ARN)" \ 
+		-var "command=$(ANSIBLE_PLAYBOOK)" \
+		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
+		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
diff --git a/.ci/ansible/requirements.txt b/.ci/ansible/requirements.txt
index eb85382ead..be0d348bcd 100644
--- a/.ci/ansible/requirements.txt
+++ b/.ci/ansible/requirements.txt
@@ -1,6 +1,5 @@
 ###### Requirements without Version Specifiers ######
 ansible
-ansible-playbook
 boto
 boto3
 opentelemetry-api
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 82302f5ee2..e502fe0e1a 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -27,6 +27,16 @@ variable "ami_suffix" {
   default = "test_suffix"
 }
 
+variable "galaxy_command" {
+  type  = string
+  default = "ansible-galaxy"
+}
+
+variable "command" {
+  type  = string
+  default = "ansible-playbook"
+}
+
 locals {
   aws_region       = "us-east-2"
   force_deregister = true
@@ -309,5 +319,7 @@ build {
     playbook_file    = "ansible/playbook.yml"
     extra_arguments  = ["--tags", "setup-ami", "--extra-vars", "nodeShellType=cmd"]
     galaxy_file      = "ansible/requirements.yml"
+    galaxy_command   = var.galaxy_command
+    command          = var.command
   }
 }
\ No newline at end of file

From fe3b0552dbc32c30ab23992b28c7376d461e7e3e Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 8 Dec 2022 17:19:07 +0100
Subject: [PATCH 51/78] Added mandatory tags

---
 .ci/packer/aws-runners.pkr.hcl | 155 ++++++++++++++++++---------------
 1 file changed, 86 insertions(+), 69 deletions(-)

diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index e502fe0e1a..5bc28160e8 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -18,7 +18,7 @@ variable "source_set" {
 }
 
 variable "org_arn" {
-  type  = string
+  type    = string
   default = ""
 }
 
@@ -28,12 +28,12 @@ variable "ami_suffix" {
 }
 
 variable "galaxy_command" {
-  type  = string
+  type    = string
   default = "ansible-galaxy"
 }
 
 variable "command" {
-  type  = string
+  type    = string
   default = "ansible-playbook"
 }
 
@@ -51,7 +51,7 @@ locals {
       "source.amazon-ebs.oracle-linux-8",
       "source.amazon-ebs.sles15"
     ],
-    "test" = ["source.amazon-ebs.ubuntu"],
+    "test"    = ["source.amazon-ebs.ubuntu"],
     "windows" = ["source.amazon-ebs.windows2019"],
     "all" = [
       "source.amazon-ebs.ubuntu",
@@ -65,6 +65,13 @@ locals {
       "source.amazon-ebs.windows2019"
     ]
   }
+  common_tags = {
+    Division = "engineering"
+    Org      = "obs"
+    Team     = "observability-robots"
+    Project  = "e2e-testing",
+    Branch   = var.ami_suffix
+  }
 }
 
 source "amazon-ebs" "ubuntu" {
@@ -80,14 +87,15 @@ source "amazon-ebs" "ubuntu" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "Ubuntu"
-    Release    = "22.04"
-    Arch       = "AMD64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Ubuntu"
+      Release    = "22.04"
+      Arch       = "AMD64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -105,14 +113,15 @@ source "amazon-ebs" "debian-10-amd64" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "Debian"
-    Release    = "10"
-    Arch       = "AMD64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Debian"
+      Release    = "10"
+      Arch       = "AMD64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -128,16 +137,17 @@ source "amazon-ebs" "debian-10-arm64" {
     device_name           = "/dev/sda1"
     volume_size           = 15
     volume_type           = "gp3"
-    delete_on_termination = true    
-  }
-  tags = {
-    OS_Version = "Debian"
-    Release    = "10"
-    Arch       = "ARM64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
+    delete_on_termination = true
   }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Debian"
+      Release    = "10"
+      Arch       = "ARM64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -155,14 +165,15 @@ source "amazon-ebs" "debian-11-amd64" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "Debian"
-    Release    = "11"
-    Arch       = "AMD64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Debian"
+      Release    = "11"
+      Arch       = "AMD64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -180,14 +191,15 @@ source "amazon-ebs" "centos-8-amd64" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "Centos"
-    Release    = "8"
-    Arch       = "AMD64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Centos"
+      Release    = "8"
+      Arch       = "AMD64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -205,14 +217,15 @@ source "amazon-ebs" "centos-8-arm64" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "Centos"
-    Release    = "8"
-    Arch       = "ARM64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Centos"
+      Release    = "8"
+      Arch       = "ARM64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -230,14 +243,15 @@ source "amazon-ebs" "oracle-linux-8" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "Oracle Linux"
-    Release    = "8"
-    Arch       = "x86-64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Oracle Linux"
+      Release    = "8"
+      Arch       = "x86-64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 
@@ -256,14 +270,15 @@ source "amazon-ebs" "sles15" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "SUSE Linux Enterprise Server 15 SP3"
-    Release    = "8"
-    Arch       = "ARM64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
-  ami_org_arns = [var.org_arn]
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "SUSE Linux Enterprise Server 15 SP3"
+      Release    = "8"
+      Arch       = "ARM64"
+    }
+  )}"
+  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -279,6 +294,8 @@ build {
     playbook_file    = "ansible/playbook.yml"
     extra_arguments  = ["--tags", "setup-ami"]
     galaxy_file      = "ansible/requirements.yml"
+    galaxy_command   = var.galaxy_command
+    command          = var.command
   }
 }
 

From 1472b8633ba8e6ca3558c38b62ae4d690c80f5a4 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 8 Dec 2022 17:39:33 +0100
Subject: [PATCH 52/78] Fix makefile

---
 .ci/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 997ff833a8..1252b9efef 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -262,7 +262,7 @@ build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)		
 	packer build \
 		-var "ami_suffix=$(AMI_SUFFIX)" \
-		-var "org_arn=$(AWS_ORG_ARN)" \ 
-		-var "command=$(ANSIBLE_PLAYBOOK)" \
+		-var "org_arn=$(AWS_ORG_ARN)" \
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
+		-var "command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl

From 6b8cd6819e9e5ab6836f9c38c8474addaaaee2e7 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 8 Dec 2022 18:02:06 +0100
Subject: [PATCH 53/78] Fixed packer ansible command variables

---
 .ci/packer/aws-runners.pkr.hcl | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 5bc28160e8..adda1721db 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -294,8 +294,8 @@ build {
     playbook_file    = "ansible/playbook.yml"
     extra_arguments  = ["--tags", "setup-ami"]
     galaxy_file      = "ansible/requirements.yml"
-    galaxy_command   = var.galaxy_command
-    command          = var.command
+    galaxy_command   = "${var.galaxy_command}"
+    command          = "${var.command}"
   }
 }
 
@@ -313,13 +313,16 @@ source "amazon-ebs" "windows2019" {
     volume_type           = "gp3"
     delete_on_termination = true
   }
-  tags = {
-    OS_Version = "Windows"
-    Release    = "2019"
-    Arch       = "x86_64"
-    Branch     = var.ami_suffix
-    Project    = "e2e"
-  }
+  tags = "${merge(
+    local.common_tags,
+    {
+      OS_Version = "Windows"
+      Release    = "2019"
+      Arch       = "x86_64"
+      Branch     = var.ami_suffix
+      Project    = "e2e"
+    }
+  )}"
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -336,7 +339,7 @@ build {
     playbook_file    = "ansible/playbook.yml"
     extra_arguments  = ["--tags", "setup-ami", "--extra-vars", "nodeShellType=cmd"]
     galaxy_file      = "ansible/requirements.yml"
-    galaxy_command   = var.galaxy_command
-    command          = var.command
+    galaxy_command   = "${var.galaxy_command}"
+    command          = "${var.command}"
   }
 }
\ No newline at end of file

From 5eca961895a4f45f16b73c823f0b5d64a1b442bc Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 8 Dec 2022 18:14:37 +0100
Subject: [PATCH 54/78] Fixed packer ansible command variables

---
 .ci/Makefile                   | 2 +-
 .ci/packer/aws-runners.pkr.hcl | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 1252b9efef..d0bf068894 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -264,5 +264,5 @@ build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 		-var "ami_suffix=$(AMI_SUFFIX)" \
 		-var "org_arn=$(AWS_ORG_ARN)" \
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
-		-var "command=$(ANSIBLE_PLAYBOOK)" \
+		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index adda1721db..251d007c1f 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -32,7 +32,7 @@ variable "galaxy_command" {
   default = "ansible-galaxy"
 }
 
-variable "command" {
+variable "playbook_command" {
   type    = string
   default = "ansible-playbook"
 }
@@ -295,7 +295,7 @@ build {
     extra_arguments  = ["--tags", "setup-ami"]
     galaxy_file      = "ansible/requirements.yml"
     galaxy_command   = "${var.galaxy_command}"
-    command          = "${var.command}"
+    command          = "${var.playbook_command}"
   }
 }
 
@@ -340,6 +340,6 @@ build {
     extra_arguments  = ["--tags", "setup-ami", "--extra-vars", "nodeShellType=cmd"]
     galaxy_file      = "ansible/requirements.yml"
     galaxy_command   = "${var.galaxy_command}"
-    command          = "${var.command}"
+    command          = "${var.playbook_command}"
   }
 }
\ No newline at end of file

From aff2207c23d15dd0501ca10e6ba2eb42e5a41517 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 8 Dec 2022 18:23:17 +0100
Subject: [PATCH 55/78] debug paths

---
 .ci/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index d0bf068894..4b010d246b 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -255,11 +255,13 @@ packer-init:
 
 .PHONY: validate-packer-template
 validate-packer-template:	
+	echo "galaxy path: $(ANSIBLE_GALAXY)"
+	echo "playbook path: $(ANSIBLE_PLAYBOOK)"
 	packer validate $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
 
 .PHONY: validate-packer-template 
 build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
-	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)		
+	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)
 	packer build \
 		-var "ami_suffix=$(AMI_SUFFIX)" \
 		-var "org_arn=$(AWS_ORG_ARN)" \

From 6c4b408b375119daf65f1e27203dc98bbf095aab Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 12 Dec 2022 13:16:52 +0100
Subject: [PATCH 56/78] Debug ansible playbook path

---
 .ci/Makefile                   | 3 +--
 .ci/packer/aws-runners.pkr.hcl | 4 ++++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 4b010d246b..3cac87e91f 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -262,8 +262,7 @@ validate-packer-template:
 .PHONY: validate-packer-template 
 build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)
-	packer build \
-		-var "ami_suffix=$(AMI_SUFFIX)" \
+	packer build -var "ami_suffix=$(AMI_SUFFIX)" \
 		-var "org_arn=$(AWS_ORG_ARN)" \
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 251d007c1f..3ccd549a5e 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -288,6 +288,10 @@ build {
 
   sources = local.source_sets[var.source_set]
 
+  provisioner "shell" {
+    inline = ["echo ${var.playbook_command}"]
+  }
+
   provisioner "ansible" {
     user             = build.User
     ansible_env_vars = ["PACKER_BUILD_NAME={{ build_name }}"]

From 9b4543cdace97452109bd2773c9767795a189488 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 20 Dec 2022 14:05:37 +0000
Subject: [PATCH 57/78] packer: Unquoted command variables

---
 .ci/packer/aws-runners.pkr.hcl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 3ccd549a5e..5abce94471 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -298,8 +298,8 @@ build {
     playbook_file    = "ansible/playbook.yml"
     extra_arguments  = ["--tags", "setup-ami"]
     galaxy_file      = "ansible/requirements.yml"
-    galaxy_command   = "${var.galaxy_command}"
-    command          = "${var.playbook_command}"
+    galaxy_command   = var.galaxy_command
+    command          = var.playbook_command
   }
 }
 
@@ -343,7 +343,7 @@ build {
     playbook_file    = "ansible/playbook.yml"
     extra_arguments  = ["--tags", "setup-ami", "--extra-vars", "nodeShellType=cmd"]
     galaxy_file      = "ansible/requirements.yml"
-    galaxy_command   = "${var.galaxy_command}"
-    command          = "${var.playbook_command}"
+    galaxy_command   = var.galaxy_command
+    command          = var.playbook_command
   }
 }
\ No newline at end of file

From ed4e9b551ae80857ee69df69a9a6fb1ad3e2d6b5 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 20 Dec 2022 14:50:33 +0000
Subject: [PATCH 58/78] Makefile: added vars to packer validate

---
 .ci/Makefile | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 3cac87e91f..dc5356ca2f 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -257,12 +257,18 @@ packer-init:
 validate-packer-template:	
 	echo "galaxy path: $(ANSIBLE_GALAXY)"
 	echo "playbook path: $(ANSIBLE_PLAYBOOK)"
-	packer validate $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
+	packer validate \
+		-var "ami_suffix=$(AMI_SUFFIX)" \
+		-var "org_arn=$(AWS_ORG_ARN)" \
+		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
+		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
+		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
 
 .PHONY: validate-packer-template 
 build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)
-	packer build -var "ami_suffix=$(AMI_SUFFIX)" \
+	packer build \
+		-var "ami_suffix=$(AMI_SUFFIX)" \
 		-var "org_arn=$(AWS_ORG_ARN)" \
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \

From a07e3ac89fc466e9cf5fc4ead475aabcd073b705 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 20 Dec 2022 15:12:16 +0000
Subject: [PATCH 59/78] packer: remover org_arn parameter

---
 .ci/Makefile                   |  6 ++----
 .ci/packer/aws-runners.pkr.hcl | 13 -------------
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index dc5356ca2f..6e85313de6 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -258,8 +258,7 @@ validate-packer-template:
 	echo "galaxy path: $(ANSIBLE_GALAXY)"
 	echo "playbook path: $(ANSIBLE_PLAYBOOK)"
 	packer validate \
-		-var "ami_suffix=$(AMI_SUFFIX)" \
-		-var "org_arn=$(AWS_ORG_ARN)" \
+		-var "ami_suffix=$(AMI_SUFFIX)" \		
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
@@ -268,8 +267,7 @@ validate-packer-template:
 build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)
 	packer build \
-		-var "ami_suffix=$(AMI_SUFFIX)" \
-		-var "org_arn=$(AWS_ORG_ARN)" \
+		-var "ami_suffix=$(AMI_SUFFIX)" \		
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 5abce94471..9f19879883 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -17,11 +17,6 @@ variable "source_set" {
   default = "linux"
 }
 
-variable "org_arn" {
-  type    = string
-  default = ""
-}
-
 variable "ami_suffix" {
   type    = string
   default = "test_suffix"
@@ -95,7 +90,6 @@ source "amazon-ebs" "ubuntu" {
       Arch       = "AMD64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -121,7 +115,6 @@ source "amazon-ebs" "debian-10-amd64" {
       Arch       = "AMD64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -147,7 +140,6 @@ source "amazon-ebs" "debian-10-arm64" {
       Arch       = "ARM64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -173,7 +165,6 @@ source "amazon-ebs" "debian-11-amd64" {
       Arch       = "AMD64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -199,7 +190,6 @@ source "amazon-ebs" "centos-8-amd64" {
       Arch       = "AMD64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -225,7 +215,6 @@ source "amazon-ebs" "centos-8-arm64" {
       Arch       = "ARM64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }
@@ -251,7 +240,6 @@ source "amazon-ebs" "oracle-linux-8" {
       Arch       = "x86-64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 
@@ -278,7 +266,6 @@ source "amazon-ebs" "sles15" {
       Arch       = "ARM64"
     }
   )}"
-  ami_org_arns     = [var.org_arn]
   skip_create_ami  = var.skip_create_ami
   force_deregister = local.force_deregister
 }

From 6e7ac2ad2aa324ae3e62e3cf106d87d55b6bd3a2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 20 Dec 2022 15:21:48 +0000
Subject: [PATCH 60/78] packer: remover org_arn parameter

---
 .ci/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 6e85313de6..5652e59bfc 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -258,7 +258,7 @@ validate-packer-template:
 	echo "galaxy path: $(ANSIBLE_GALAXY)"
 	echo "playbook path: $(ANSIBLE_PLAYBOOK)"
 	packer validate \
-		-var "ami_suffix=$(AMI_SUFFIX)" \		
+		-var "ami_suffix=$(AMI_SUFFIX)" \
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
@@ -267,7 +267,7 @@ validate-packer-template:
 build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)
 	packer build \
-		-var "ami_suffix=$(AMI_SUFFIX)" \		
+		-var "ami_suffix=$(AMI_SUFFIX)" \
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl

From 1b269363797db6ade658fbe3957c53220049d652 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 21 Dec 2022 15:04:06 +0000
Subject: [PATCH 61/78] added ami suffix specifier

---
 .ci/Jenkinsfile | 2 +-
 .ci/Makefile    | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 1e360accf9..75f5a7b084 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -152,7 +152,7 @@ pipeline {
               deleteDir()
               unstash 'source'
               script {
-                e2eUtils.buildPackerAMIs(amiSuffix: 'main')
+                e2eUtils.buildPackerAMIs(amiSuffix: env.CHANGE_ID)
               }
             }
           }          
diff --git a/.ci/Makefile b/.ci/Makefile
index 5652e59bfc..9744bb5423 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -263,7 +263,7 @@ validate-packer-template:
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
 
-.PHONY: validate-packer-template 
+.PHONY: build-amis-$(AMI_SUFFIX)
 build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)
 	packer build \
@@ -271,3 +271,9 @@ build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 		-var "galaxy_command=$(ANSIBLE_GALAXY)" \
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
+
+
+# .PHONY: deregister-amis-$(AMI_SUFFIX)
+# deregister-amis-$(AMI_SUFFIX)		 
+# 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for deregister)
+	

From 06664c69e8f7de09de6213e8e5b63f3e6f5bba40 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Dec 2022 16:02:35 +0000
Subject: [PATCH 62/78] amis deletion test

---
 .ci/Jenkinsfile                       |  2 ++
 .ci/Makefile                          | 14 ++++++++++----
 .ci/ansible/delete-amis.yml           |  6 ++++++
 .ci/ansible/tasks/deregister-amis.yml | 17 +++++++++++++++++
 .ci/e2eUtils.groovy                   | 13 +++++++++++++
 5 files changed, 48 insertions(+), 4 deletions(-)
 create mode 100644 .ci/ansible/delete-amis.yml
 create mode 100644 .ci/ansible/tasks/deregister-amis.yml

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 75f5a7b084..62918c8eac 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -226,6 +226,7 @@ pipeline {
               script {                
                 def testMatrix = readYaml(file: "${env.REAL_BASE_DIR}/${TEST_MATRIX_FILE}")
                 e2eUtils.runE2ETests(
+                  amiSuffix: env.CHANGE_ID,
                   testMatrix: testMatrix,
                   selectedSuites: env.RUN_TEST_SUITES,
                   runAsMainBranch: env.RUN_AS_MAIN_BRANCH,
@@ -270,6 +271,7 @@ pipeline {
   post {
     cleanup {
       script {
+        e2eUtils.deregisterAMIs(amisRebuilt: env.REBUILD_AMIS)
         e2eUtils.doNotifyBuildResult(
           slackNotify: params.notifyOnGreenBuilds,
           slackChannel: env.SLACK_CHANNEL,
diff --git a/.ci/Makefile b/.ci/Makefile
index 9744bb5423..3ade00e196 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -272,8 +272,14 @@ build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
 		-var "playbook_command=$(ANSIBLE_PLAYBOOK)" \
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
 
+.PHONY: sanitize-ami-suffix
+sanitize-ami-suffix:
+ifeq ($(AMI_SUFFIX),main)
+	echo "AMI_SUFFIX is main. We do not delete main AMIs"; exit 1 		
+endif
+
+.PHONY: deregister-amis-$(AMI_SUFFIX)
+deregister-amis-$(AMI_SUFFIX): sanitize-ami-suffix
+	@:$(call check_defined, AMI_SUFFIX, AMI_SUFFIX must be defined for deregister)
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/delete-amis.yml --extra-vars "amiSuffix=$(AMI_SUFFIX)" 
 
-# .PHONY: deregister-amis-$(AMI_SUFFIX)
-# deregister-amis-$(AMI_SUFFIX)		 
-# 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for deregister)
-	
diff --git a/.ci/ansible/delete-amis.yml b/.ci/ansible/delete-amis.yml
new file mode 100644
index 0000000000..6ccf65c226
--- /dev/null
+++ b/.ci/ansible/delete-amis.yml
@@ -0,0 +1,6 @@
+- name: Delete temporary AMIs
+  hosts: localhost
+  gather_facts: True    
+  tasks:
+  - name: Deregister AMIs and delete snapshots
+    include_tasks: tasks/deregister-amis.yml
diff --git a/.ci/ansible/tasks/deregister-amis.yml b/.ci/ansible/tasks/deregister-amis.yml
new file mode 100644
index 0000000000..121dab3b91
--- /dev/null
+++ b/.ci/ansible/tasks/deregister-amis.yml
@@ -0,0 +1,17 @@
+---
+
+- name: find AMIs by tag
+  amazon.aws.ec2_ami_info:
+    region: "us-east-2"
+    filters:
+      "tag:Branch": '{{amiSuffix}}'
+      "tag:Project": 'e2e-testing'
+  register: amis_to_delete
+   
+- name: Deregister AMIs and delete associated snapshots
+  amazon.aws.ec2_ami:
+    region: "us-east-2"
+    image_id: "{{ item.image_id }}"
+    delete_snapshot: True
+    state: absent
+  loop: "{{amis_to_delete.images}}"
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index fac720dbdc..9467b8cd2f 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -368,6 +368,19 @@ def buildPackerAMIs(Map args = [:]) {
     }
 }
 
+def deregisterAMIs(Map args = [:]) {
+    if (!args.amisResuilt) return
+    dir("${BASE_DIR}") {
+        if (!args.amiSuffix?.trim()) {
+            error("amiSuffix parameter must be specified in deregisterAMIs()")
+        }
+        setEnvVar("AMI_SUFFIX", args.amiSuffix)
+        ciBuild() {            
+            sh(label: "Deregister AMIs with tagged by Branch :${args.amiSuffix}", script: 'make -C .ci deregister-amis-$AMI_SUFFIX')
+        }
+    }
+}
+
 def retryWithNode(Map args = [:], Closure body) {
     try {
         incrementRetries(args.stageName)

From 4fcd1b7599c2265b6e3306f429f9ff0f0a55a1d4 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Dec 2022 14:17:11 +0000
Subject: [PATCH 63/78] ansible: converted delete amis to a single playbook

---
 .ci/Jenkinsfile             |  4 ++--
 .ci/ansible/delete-amis.yml | 17 +++++++++++++++--
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 62918c8eac..e1da32ef06 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -242,6 +242,7 @@ pipeline {
               // Once all tests are complete we need to teardown the single instance with the deployed stack
               script {
                 e2eUtils.destroyStack(destroyCloudResources: env.DESTROY_CLOUD_RESOURCES)
+                e2eUtils.deregisterAMIs(amisRebuilt: env.REBUILD_AMIS)
               }
             }
           }
@@ -270,8 +271,7 @@ pipeline {
   }
   post {
     cleanup {
-      script {
-        e2eUtils.deregisterAMIs(amisRebuilt: env.REBUILD_AMIS)
+      script {        
         e2eUtils.doNotifyBuildResult(
           slackNotify: params.notifyOnGreenBuilds,
           slackChannel: env.SLACK_CHANNEL,
diff --git a/.ci/ansible/delete-amis.yml b/.ci/ansible/delete-amis.yml
index 6ccf65c226..888d5c6480 100644
--- a/.ci/ansible/delete-amis.yml
+++ b/.ci/ansible/delete-amis.yml
@@ -2,5 +2,18 @@
   hosts: localhost
   gather_facts: True    
   tasks:
-  - name: Deregister AMIs and delete snapshots
-    include_tasks: tasks/deregister-amis.yml
+  - name: find AMIs by tag
+    amazon.aws.ec2_ami_info:
+      region: "us-east-2"
+      filters:
+        "tag:Branch": '{{amiSuffix}}'
+        "tag:Project": 'e2e-testing'
+    register: amis_to_delete
+   
+  - name: Deregister AMIs and delete associated snapshots
+    amazon.aws.ec2_ami:
+      region: "us-east-2"
+      image_id: "{{ item.image_id }}"
+      delete_snapshot: True
+      state: absent
+    loop: "{{amis_to_delete.images}}"

From b8dccb063fffd28e54b444fab7ebd014fc8389a8 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Dec 2022 15:11:48 +0000
Subject: [PATCH 64/78] Jenkinsfile: Added internal post stage to deregister
 amis

---
 .ci/Jenkinsfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index e1da32ef06..e0aeee5b27 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -242,7 +242,6 @@ pipeline {
               // Once all tests are complete we need to teardown the single instance with the deployed stack
               script {
                 e2eUtils.destroyStack(destroyCloudResources: env.DESTROY_CLOUD_RESOURCES)
-                e2eUtils.deregisterAMIs(amisRebuilt: env.REBUILD_AMIS)
               }
             }
           }
@@ -267,6 +266,11 @@ pipeline {
           }
         }
       }
+      post {
+        always {
+          e2eUtils.deregisterAMIs(amisRebuilt: env.REBUILD_AMIS)
+        }
+      }
     }
   }
   post {

From 39e00f844f8ae66e2737858c8299cb317db9f70c Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Dec 2022 15:16:57 +0000
Subject: [PATCH 65/78] Jenkinsfile: Added internal post stage to deregister
 amis

---
 .ci/Jenkinsfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index e0aeee5b27..dd681bf7b7 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -268,7 +268,9 @@ pipeline {
       }
       post {
         always {
-          e2eUtils.deregisterAMIs(amisRebuilt: env.REBUILD_AMIS)
+          script {
+            e2eUtils.deregisterAMIs(amisRebuilt: env.REBUILD_AMIS)
+          }
         }
       }
     }

From 8aec03cfa0070b3134388ebfd777b429c25addee Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Dec 2022 15:43:05 +0000
Subject: [PATCH 66/78] packer: temporary commented sles15 due to a problem

---
 .ci/packer/aws-runners.pkr.hcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 9f19879883..7752162a21 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -44,7 +44,7 @@ locals {
       "source.amazon-ebs.centos-8-amd64",
       "source.amazon-ebs.centos-8-arm64",
       "source.amazon-ebs.oracle-linux-8",
-      "source.amazon-ebs.sles15"
+      # "source.amazon-ebs.sles15"
     ],
     "test"    = ["source.amazon-ebs.ubuntu"],
     "windows" = ["source.amazon-ebs.windows2019"],
@@ -56,7 +56,7 @@ locals {
       "source.amazon-ebs.centos-8-amd64",
       "source.amazon-ebs.centos-8-arm64",
       "source.amazon-ebs.oracle-linux-8",
-      "source.amazon-ebs.sles15",
+      # "source.amazon-ebs.sles15",
       "source.amazon-ebs.windows2019"
     ]
   }

From 046dc4c9d78880c465fc2e137588041d83a1b3a0 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 20 Jan 2023 15:15:20 +0000
Subject: [PATCH 67/78] changed deregister condition

---
 .ci/Makefile                   |  3 ++-
 .ci/ansible/test.yml           | 19 +++++++++++++++++++
 .ci/e2eUtils.groovy            |  2 +-
 .ci/packer/aws-runners.pkr.hcl |  2 +-
 4 files changed, 23 insertions(+), 3 deletions(-)
 create mode 100644 .ci/ansible/test.yml

diff --git a/.ci/Makefile b/.ci/Makefile
index 3ade00e196..a20f5f177b 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -142,6 +142,7 @@ show-stack:
 provision-node: .runID set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
+	echo "mk nodeImage: $${NODE_IMAGE}"
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml --user $(NODE_USER) \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
@@ -229,7 +230,7 @@ run-tests: set-env-$(NODE_LABEL)
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && PROVIDER="$(PROVIDER)" SUITE="$(SUITE)" TAGS="$(TAGS)" REPORT_PREFIX="$(SUITE)_$${NODE_LABEL}_$(TAGS)" \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml --user $(NODE_USER) \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
+		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$(NODE_IMAGE) nodeInstanceType=$${NODE_INSTANCE_TYPE}" \
 		--extra-vars "runId=$(RUN_ID) nodeShellType=$${NODE_SHELL_TYPE} workspace=$(PROJECT_DIR)/ suite=$(SUITE) sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t run-tests \
diff --git a/.ci/ansible/test.yml b/.ci/ansible/test.yml
new file mode 100644
index 0000000000..d6d8f882d1
--- /dev/null
+++ b/.ci/ansible/test.yml
@@ -0,0 +1,19 @@
+- name: test
+  hosts: localhost
+  gather_facts: True    
+  tasks:
+  - name: transform AMI name to id
+    amazon.aws.ec2_ami_info:
+      region: "us-east-2"
+      filters:
+        name: '{{nodeImage}}'
+    register: preinstalled_images
+    tags:    
+      - provision-node
+      - provision-stack
+  - name: Print found images
+    ansible.builtin.debug:
+      var: preinstalled_images
+    tags:
+      - provision-node
+      - provision-stack      
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index 9467b8cd2f..2f0bd689c9 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -369,7 +369,7 @@ def buildPackerAMIs(Map args = [:]) {
 }
 
 def deregisterAMIs(Map args = [:]) {
-    if (!args.amisResuilt) return
+    if (args.amisRebuilt? != "true") return
     dir("${BASE_DIR}") {
         if (!args.amiSuffix?.trim()) {
             error("amiSuffix parameter must be specified in deregisterAMIs()")
diff --git a/.ci/packer/aws-runners.pkr.hcl b/.ci/packer/aws-runners.pkr.hcl
index 7752162a21..76b22dcdcc 100644
--- a/.ci/packer/aws-runners.pkr.hcl
+++ b/.ci/packer/aws-runners.pkr.hcl
@@ -57,7 +57,7 @@ locals {
       "source.amazon-ebs.centos-8-arm64",
       "source.amazon-ebs.oracle-linux-8",
       # "source.amazon-ebs.sles15",
-      "source.amazon-ebs.windows2019"
+      # "source.amazon-ebs.windows2019"
     ]
   }
   common_tags = {

From a303538cc5bf4fb03f5616ec779bda86930922bb Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 23 Jan 2023 15:35:53 +0000
Subject: [PATCH 68/78] fixed merge mistakes

---
 .ci/Jenkinsfile | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 6a6dca0024..3594062226 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -182,29 +182,15 @@ pipeline {
                 sh "ssh-keygen -b 4096 -t rsa -f ${E2E_SSH_KEY} -q -N \"\" "
 
                 dir("${env.REAL_BASE_DIR}") {
-<<<<<<< HEAD
-                  e2eUtils.ciBuild() {
-                    // sh(label: 'Create Stack node', script: "make -C .ci provision-stack")
-                  }
-                  e2eUtils.ciBuild() {
-                    retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                      // sh(label: 'Setup Stack node', script: "make -C .ci setup-stack")
-                    }
-                  }
-                  e2eUtils.ciBuild() {
-                    retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                      // sh(label: 'Start Elastic Stack', script: "make -C .ci start-elastic-stack")
-=======
                   withEnv([
                     "SUITE=fleet",
                     "STACK_INSTANCE_ID=${env.BUILD_URL}_stack",
                     "TAGS=non-existing-tag"
                   ]) {
-                    ciBuild() {
+                    e2eUtils.ciBuild() {
                       retryWithSleep(retries: 3, seconds: 5, backoff: true){
                         sh(label: 'Setup Stack node', script: "make -C .ci create-stack")
                       }
->>>>>>> f60f1909bdb22ef7c24ea2aae5b090490a888673
                     }
                   }
 

From 9016d37592a75a77ec5a6c725bc8dc1a367fce69 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 23 Jan 2023 15:48:44 +0000
Subject: [PATCH 69/78] fixed merge mistakes

---
 .ci/Jenkinsfile     | 339 --------------------------------------------
 .ci/e2eUtils.groovy |  26 +---
 2 files changed, 5 insertions(+), 360 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 3594062226..7fd8be9a94 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -274,7 +274,6 @@ pipeline {
   }
   post {
     cleanup {
-<<<<<<< HEAD
       script {        
         e2eUtils.doNotifyBuildResult(
           slackNotify: params.notifyOnGreenBuilds,
@@ -284,346 +283,8 @@ pipeline {
           githubCheckSha1: env.GITHUB_CHECK_SHA1,
           githubCheckName: env.GITHUB_GITHUB_CHECK_NAME
         )
-=======
-      doNotifyBuildResult(params.notifyOnGreenBuilds)
-    }
-  }
-}
-
-// this function evaluates if the test stage of the build must be executed
-def checkSkipTests() {
-  dir("${BASE_DIR}"){
-
-    // only docs means no tests are run
-    if (isGitRegionMatch(patterns: [ '.*\\.md' ], shouldMatchAll: true)) {
-      setEnvVar("SKIP_TESTS", true)
-      return
-    }
-
-    // patterns for all places that should trigger a full build
-    def regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
-    setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: regexps, shouldMatchAll: false))
-  }
-}
-
-/*
- * Runs the Make build at the CI, executing the closure in the context of Ansible + AWS
- */
-def ciBuild(Closure body){
-  withEnv([
-    "SSH_KEY=${E2E_SSH_KEY}"
-  ]) {
-    def awsProps = getVaultSecret(secret: "${AWS_PROVISIONER_SECRET}")
-    def awsAuthObj = awsProps?.data
-    withEnv([
-      "ANSIBLE_CONFIG=${env.REAL_BASE_DIR}/.ci/ansible/ansible.cfg",
-      "ANSIBLE_HOST_KEY_CHECKING=False",
-    ]){
-      withVaultToken(){
-        withEnvMask(vars: [
-          [var: "AWS_ACCESS_KEY_ID", password: awsAuthObj.access_key],
-          [var: "AWS_SECRET_ACCESS_KEY", password: awsAuthObj.secret_key]
-        ]) {
-          withOtelEnv() {
-            body()
-          }
-        }
->>>>>>> f60f1909bdb22ef7c24ea2aae5b090490a888673
       }
     }
   }
 }
 
-<<<<<<< HEAD
-=======
-def getNodeIp(nodeType){
-  return sh(label: "Get IP address of the ${nodeType}", script: "cat ${REAL_BASE_DIR}/.ci/.${nodeType}-host-ip", returnStdout: true)
-}
-
-def getRemoteE2EPath(testRunner, platform) {
-  if (platform.contains("windows")) {
-    return "C:\\Users\\${testRunner.user}\\e2e-testing\\"
-  }
-
-  return "/home/${testRunner.user}/e2e-testing/"
-}
-
-def getMachineInfo(platform){
-  def machineYaml = readYaml(file: "${env.REAL_BASE_DIR}/.ci/.e2e-platforms.yaml")
-  def machines = machineYaml['PLATFORMS']
-  log(level: 'INFO', text: "getMachineInfo: machines.get(platform)=${machines.get(platform)}")
-  return machines.get(platform)
-}
-
-def checkTestSuite(Map parallelTasks = [:], Map item = [:]) {
-  def suite = item.suite
-  def platforms = item.platforms
-
-  // Predefine the remote provider to use the already provisioned stack VM.
-  // Each suite or scenario in the CI test suite would be able to define its own provider
-  // (i.e. docker). If empty, remote will be used as fallback
-  def suiteProvider = item.provider
-  if (!suiteProvider || suiteProvider?.trim() == '') {
-    suiteProvider = 'remote'
-  }
-
-  item.scenarios.each { scenario ->
-    def name = scenario.name
-    def platformsValue = platforms
-
-    def scenarioProvider = scenario.provider
-    // if the scenario does not set its own provider, use suite's provider
-    if (!scenarioProvider || scenarioProvider?.trim() == '') {
-      scenarioProvider = suiteProvider
-    }
-
-    def scenarioPlatforms = scenario.platforms
-    if (scenarioPlatforms?.size() > 0) {
-      // scenario platforms take precedence over suite platforms, overriding them
-      platformsValue = scenarioPlatforms
-    }
-    def pullRequestFilter = scenario.containsKey('pullRequestFilter') ? scenario.pullRequestFilter : ''
-    def tags = scenario.tags
-      platformsValue.each { rawPlatform ->
-        // platform is not space based, so let's ensure no extra spaces can cause misbehaviours.
-        def platform = rawPlatform.trim()
-        log(level: 'INFO', text: "Adding ${suite}:${platform}:${tags} test suite to the build execution")
-        def machineInfo = getMachineInfo(platform)
-        def stageName = "${suite}_${platform}_${tags}"
-        parallelTasks["${stageName}"] = generateFunctionalTestStep(name: "${name}",
-                                                                  platform: platform,
-                                                                  provider: scenarioProvider,
-                                                                  suite: "${suite}",
-                                                                  tags: "${tags}",
-                                                                  pullRequestFilter: "${pullRequestFilter}",
-                                                                  machine: machineInfo,
-                                                                  stageName: stageName)
-      }
-  }
-}
-
-/*
- * Sends out notification of the build result to Slack
- */
-def doNotifyBuildResult(boolean slackNotify) {
-  def doSlackNotify = true // always try to notify on failures
-  def githubCheckStatus = 'FAILURE'
-  if (currentBuild.currentResult == 'SUCCESS') {
-    githubCheckStatus = 'SUCCESS'
-    doSlackNotify = slackNotify // if the build status is success, read the parameter
-  }
-
-  githubCheckNotify(githubCheckStatus)
-
-
-  def testsSuites = "${params.runTestsSuites}"
-  if (testsSuites?.trim() == "") {
-    testsSuites = "All suites"
-  }
-
-  def channels = "${env.SLACK_CHANNEL}"
-  if (channels?.trim() == "") {
-    channels = "observablt-bots"
-  }
-
-  def header = "*Test Suite*: " + testsSuites
-  notifyBuildResult(analyzeFlakey: true,
-                    jobName: getFlakyJobName(withBranch: "${env.JOB_BASE_NAME}"),
-                    prComment: true,
-                    slackHeader: header,
-                    slackChannel: "${channels}",
-                    slackComment: true,
-                    slackNotify: doSlackNotify)
-}
-
-/**
- Notify the GitHub check of the parent stream
- **/
-def githubCheckNotify(String status) {
-  if (params.GITHUB_CHECK_NAME?.trim() && params.GITHUB_CHECK_REPO?.trim() && params.GITHUB_CHECK_SHA1?.trim()) {
-    githubNotify context: "${params.GITHUB_CHECK_NAME}",
-      description: "${params.GITHUB_CHECK_NAME} ${status.toLowerCase()}",
-      status: "${status}",
-      targetUrl: "${env.RUN_DISPLAY_URL}",
-      sha: params.GITHUB_CHECK_SHA1, account: 'elastic', repo: params.GITHUB_CHECK_REPO, credentialsId: env.JOB_GIT_CREDENTIALS
-  }
-}
-
-def generateFunctionalTestStep(Map args = [:]){
-  def name = args.get('name')
-  def name_normalize = name.replace(' ', '_')
-  def platform = args.get('platform')
-  def provider = args.get('provider')
-  def suite = args.get('suite')
-  def tags = args.get('tags')
-  def pullRequestFilter = args.get('pullRequestFilter')?.trim() ? args.get('pullRequestFilter') : ''
-  def machine = args.get('machine')
-  def stageName = args.get('stageName')
-
-  // TODO: Is this still relevant?
-  if (isPR() || isUpstreamTrigger(filter: 'PR-')) {
-    // when the "Run_As_Main_Branch" param is disabled, we will honour the PR filters, which
-    // basically exclude some less frequent platforms or operative systems. If the user enabled
-    // this param, the pipeline will remove the filters from the test runner.
-    if (!params.Run_As_Main_Branch) {
-      tags += pullRequestFilter
-    }
-  }
-
-  def goArch = "amd64"
-  if (platform.contains("arm64")) {
-    goArch = "arm64"
-  }
-
-  // sanitize tags to create the file
-  def sanitisedTags = tags.replaceAll("\\s","_")
-  sanitisedTags = sanitisedTags.replaceAll("~","")
-  sanitisedTags = sanitisedTags.replaceAll("@","")
-
-  def githubCheckSha1 = params.GITHUB_CHECK_SHA1?.trim() ? params.GITHUB_CHECK_SHA1 : ''
-  def githubCheckRepo = params.GITHUB_CHECK_REPO?.trim() ? params.GITHUB_CHECK_REPO : ''
-
-  // Setup environment for platform
-  def envContext = []
-  envContext.add("PROVIDER=${provider}")
-  envContext.add("GITHUB_CHECK_SHA1=${githubCheckSha1}")
-  envContext.add("GITHUB_CHECK_REPO=${githubCheckRepo}")
-  envContext.add("SUITE=${suite}")
-  envContext.add("TAGS=${tags}")
-  envContext.add("REPORT_PREFIX=${suite}_${platform}_${sanitisedTags}")
-  envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},elastic_agent_version=${env.ELASTIC_AGENT_VERSION},stack_version=${env.STACK_VERSION}")
-  // VM characteristics
-  envContext.add("NODE_LABEL=${platform}")
-  envContext.add("NODE_IMAGE=${machine.image}")
-  envContext.add("NODE_INSTANCE_ID=${env.BUILD_URL}_${platform}_${suite}_${tags}")
-  envContext.add("NODE_INSTANCE_TYPE=${machine.instance_type}")
-  envContext.add("NODE_SHELL_TYPE=${machine.shell_type}")
-  envContext.add("NODE_USER=${machine.username}")
-
-  return {
-    // Set the worker as flaky for the time being, this will be changed in the finally closure.
-    setFlakyWorker(stageName)
-    retryWithNode(labels: 'ubuntu-20.04 && gobld/machineType:e2-small', forceWorkspace: true, forceWorker: true, stageName: stageName){
-      try {
-        deleteDir()
-        dir("${env.REAL_BASE_DIR}") {
-          unstash 'sourceEnvModified'
-          withEnv(envContext) {
-            // This step will help to send the APM traces to the
-            // withOtelEnv is the one that uses the APM service defined by the Otel Jenkins plugin.
-            // withAPMEnv uses Vault to prepare the context.
-            // IMPORTANT: withAPMEnv is now the one in used since withOtelEnv uses a specific Opentelemetry Collector at the moment.
-            // TODO: This will need to be integrated into the provisioned VMs
-            withAPMEnv() {
-              // we are separating the different test phases to avoid recreating
-              ciBuild() {
-                  sh(label: 'Start node', script: "make -C .ci provision-node")
-              }
-
-              // make goal to run the tests, which is platform-dependant
-              def runCommand = "run-tests"
-
-              if (platform.contains("windows")) {
-                runCommand = "run-tests-win"
-                // Ansible wait_for module is not enough to mitigate the timeout
-                log(level: 'DEBUG', text: "Sleeping 300 seconds on Windows so that SSH is accessible in the remote instance.")
-                sleep(300)
-              }
-
-              ciBuild() {
-                retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                  sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
-                }
-              }
-              ciBuild() {
-                sh(label: 'Run tests in the node', script: "make -C .ci ${runCommand}")
-              }
-            }
-          }
-        }
-      } finally {
-        withEnv(envContext) {
-          dir("${env.REAL_BASE_DIR}") {
-            // If it reaches this point then the CI worker is most likely behaving correctly
-            // there is still a chance things might fail afterwards, but this is just the finally
-            // section so we could say we are good to go.
-            // It runs after dir so if the worker is gone the an error will be thrown regarding
-            // the dir cannot be accessed in the existing none worker.
-            unsetFlakyWorker(stageName)
-            def testRunnerIP = getNodeIp("node")
-            sh "mkdir -p outputs/${testRunnerIP} || true"
-            ciBuild() {
-              sh(label: 'Fetch tests reports from node', script: "make -C .ci fetch-test-reports")
-            }
-            sh "ls -l outputs/${testRunnerIP}"
-            if (!params.DESTROY_CLOUD_RESOURCES) {
-              log(level: 'INFO', text: "Cloud instance won't be destroyed after the build. Please SSH into the test runner machine on ${testRunnerIP}.")
-            } else {
-              log(level: 'INFO', text: "Destroying Cloud instance")
-              ciBuild() {
-                retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                  sh(label: 'Destroy node', script: "make -C .ci destroy-node")
-                }
-              }
-            }
-            archiveArtifacts(allowEmptyArchive: true, artifacts: "outputs/**/TEST-*,outputs/**/*.zip,outputs/**/*.tgz")
-            junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/**/TEST-*.xml")
-          }
-        }
-      }
-    }
-  }
-}
-
-def retryWithNode(Map args = [:], Closure body) {
-  try {
-    incrementRetries(args.stageName)
-    withNode(args){
-      body()
-    }
-  } catch (err) {
-    log(level: 'WARN', text: "Stage '${args.stageName}' failed, let's analyse if it's a flaky CI worker.")
-    if (isFlakyWorker(args.stageName) && isRetryAvailable(args.stageName)) {
-      log(level: 'INFO', text: "Rerun '${args.stageName}' in a new worker.")
-      retryWithNode(args) {
-        body()
-      }
-    } else {
-      error("Error '${err.toString()}'")
-    }
-  }
-}
-
-def isFlakyWorker(stageName) {
-  if (workersStatus.containsKey(stageName)) {
-    return !workersStatus.get(stageName).get('status', true)
-  }
-  return false
-}
-
-def isRetryAvailable(stageName) {
-  return workersStatus.get(stageName).get('retries', 2) < 2
-}
-
-def incrementRetries(stageName) {
-  if (workersStatus.containsKey(stageName)) {
-    def current = workersStatus[stageName].get('retries', 0)
-    workersStatus[stageName].retries = current + 1
-  } else {
-    setFlakyWorker(stageName)
-    workersStatus[stageName].retries = 1
-  }
-}
-
-def setFlakyWorker(stageName) {
-  if (workersStatus.containsKey(stageName)) {
-    workersStatus[stageName].status = false
-  } else {
-    workersStatus[stageName] = [ status: false ]
-  }
-}
-
-def unsetFlakyWorker(stageName) {
-  workersStatus[stageName].status = true
-}
->>>>>>> f60f1909bdb22ef7c24ea2aae5b090490a888673
diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index d73b047494..d8d53fda42 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -111,12 +111,12 @@ def checkSkipTests() {
         // patterns for all places that should trigger a full build
         def tests_regexps = [ 
             "^e2e/_suites/fleet/.*", 
-            "^e2e/_suites/helm/.*", 
             "^e2e/_suites/kubernetes-autodiscover/.*", 
             "^.ci/.*", 
             "^cli/.*", 
             "^e2e/.*\\.go", 
-            "^internal/.*\\.go" ]
+            "^internal/.*\\.go" 
+        ]
         // def ami_regexps = [ "^.ci/ansible/.*", "^.ci/packer/.*"]
         setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: tests_regexps, shouldMatchAll: false))        
     }
@@ -140,10 +140,7 @@ def ciBuild(Closure body) {
                         [var: "AWS_ACCESS_KEY_ID", password: awsAuthObj.access_key],
                         [var: "AWS_SECRET_ACCESS_KEY", password: awsAuthObj.secret_key]
                 ]) {
-                    withOtelEnv() {
-                        retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                            sh("make -C .ci setup-env") // make sure the environment is created
-                        }
+                    withOtelEnv() {                        
                         body()
                     }
                 }
@@ -332,21 +329,8 @@ def generateFunctionalTestStep(Map args = [:]) {
                                 }
                             }
                         }
-                        withEnv([
-                                "ARCHITECTURE=${goArch}",
-                                "CUCUMBER_REPORTS_PATH=${env.REAL_BASE_DIR}/outputs/${testRunnerIP}",
-                                "PLATFORM=${platform}",
-                                "SUITE=${suite}",
-                                "TAGS=${tags}",
-                        ]){
-                            retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                                dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-                                sh(script: ".ci/scripts/generate-cucumber-reports.sh", label: "generate-cucumber-reports.sh")
-                            }
-                        }
-                        junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/${testRunnerIP}/TEST-*.xml")
-                        archiveArtifacts allowEmptyArchive: true,
-                                artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html"
+                        archiveArtifacts(allowEmptyArchive: true, artifacts: "outputs/**/TEST-*,outputs/**/*.zip,outputs/**/*.tgz")
+                        junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/**/TEST-*.xml")
                     }
                 }
             }

From c2782a55d44512aba65e4e9908491c720cb71cfa Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 23 Jan 2023 16:04:16 +0000
Subject: [PATCH 70/78] fixed merge mistakes

---
 .ci/e2eUtils.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/e2eUtils.groovy b/.ci/e2eUtils.groovy
index d8d53fda42..4f9d6dc6a9 100644
--- a/.ci/e2eUtils.groovy
+++ b/.ci/e2eUtils.groovy
@@ -353,7 +353,7 @@ def buildPackerAMIs(Map args = [:]) {
 }
 
 def deregisterAMIs(Map args = [:]) {
-    if (args.amisRebuilt? != "true") return
+    if (args.amisRebuilt != "true") return
     dir("${BASE_DIR}") {
         if (!args.amiSuffix?.trim()) {
             error("amiSuffix parameter must be specified in deregisterAMIs()")

From f5a2d9468dca631a1c824dc3cccb38b48510c1f2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 23 Jan 2023 16:11:27 +0000
Subject: [PATCH 71/78] fixed merge mistakes

---
 .ci/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 60e6b5e0e9..a7709c71c5 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -292,7 +292,7 @@ validate-packer-template:
 		 $(PROJECT_DIR)/.ci/packer/aws-runners.pkr.hcl
 
 .PHONY: build-amis-$(AMI_SUFFIX)
-build-amis-$(AMI_SUFFIX): packer-init validate-packer-template
+build-amis-$(AMI_SUFFIX): setup-env packer-init validate-packer-template
 	@:$(call check_defined, AMI_SUFFIX, You need to define a unique AMI_SUFFIX for testing. Or specify AMI_SUFFIX=main to override existing prodution AMIs)
 	packer build \
 		-var "ami_suffix=$(AMI_SUFFIX)" \

From 47b34fa9cc052ea98a08df1d26405c877511c9a2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 23 Jan 2023 16:34:35 +0000
Subject: [PATCH 72/78] fixed merge mistakes

---
 .ci/ansible/tasks/runners.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 23da55bb57..63a52567aa 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -50,7 +50,7 @@
     key_name: "e2essh-{{runId}}"
     region: us-east-2
     security_group: e2e
-    image_id: '{{nodeImage}}'
+    image_id: '{{ami_image}}'
     instance_type: '{{nodeInstanceType}}'
     network:
       assign_public_ip: true

From 2f7beac9359d0135fc94826f81a87d5b66a70e08 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 23 Jan 2023 16:56:58 +0000
Subject: [PATCH 73/78] fixed merge mistakes

---
 .ci/ansible/tasks/runners.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 63a52567aa..3d9af53302 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -50,7 +50,7 @@
     key_name: "e2essh-{{runId}}"
     region: us-east-2
     security_group: e2e
-    image_id: '{{ami_image}}'
+    image_id: '{{ ami_image.image_id }}'
     instance_type: '{{nodeInstanceType}}'
     network:
       assign_public_ip: true

From 60873706551490f8a5137c84c74f1a3781a97ec6 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Jan 2023 15:40:20 +0000
Subject: [PATCH 74/78] test deploy test infra

---
 .ci/Jenkinsfile               | 3 ++-
 .ci/ansible/tasks/runners.yml | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 7fd8be9a94..d8e3db8f63 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -143,7 +143,8 @@ pipeline {
           when {
             beforeAgent true
             anyOf {
-              expression { return env.REBUILD_AMIS == "true" }
+              // expression { return env.REBUILD_AMIS == "true" }
+              expression { return false }
             }
           }
           steps {
diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index 3d9af53302..f91f4042cb 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -89,10 +89,10 @@
     - provision-node
 
 - name: Add AWS host to ssh address list
-  no_log: true
+  # no_log: true
   lineinfile:
     state: present
-    line: "- {{ nodeUser }}@{{addr.public_ip_address}}"
+    line: "- {{ ansible_user }}@{{addr.public_ip_address}}"
     insertafter: EOF
     dest: "{{ workspace }}{{nodeLabel}}-sshhosts"
     create: true

From e340e950b881e17c0ca8a043b088ff041d754258 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Jan 2023 15:53:37 +0000
Subject: [PATCH 75/78] test deploy test infra

---
 .ci/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index a7709c71c5..5e9c2917c9 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -115,7 +115,7 @@ provision-stack: setup-env show-env
 	source $(VENV_DIR)/bin/activate; \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE)" \
+		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE)" nodeUser=$(STACK_USER)" \
 		--extra-vars "runId=$(RUN_ID) instanceID=$(STACK_INSTANCE_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t provision-stack

From 5e4f31f5c5f11725c26c231c096adebe0a7a2169 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Jan 2023 16:00:49 +0000
Subject: [PATCH 76/78] test deploy test infra

---
 .ci/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/Makefile b/.ci/Makefile
index 5e9c2917c9..eb26d96c74 100644
--- a/.ci/Makefile
+++ b/.ci/Makefile
@@ -115,7 +115,7 @@ provision-stack: setup-env show-env
 	source $(VENV_DIR)/bin/activate; \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
-		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE)" nodeUser=$(STACK_USER)" \
+		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
 		--extra-vars "runId=$(RUN_ID) instanceID=$(STACK_INSTANCE_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
 		--ssh-common-args='$(SSH_OPTS)' \
 		-t provision-stack

From 7dc8075d015efa65d61fd59f333c9ce15f2aa3fd Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Jan 2023 16:12:02 +0000
Subject: [PATCH 77/78] test deploy test infra

---
 .ci/ansible/tasks/runners.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index f91f4042cb..e5e15db75c 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -92,7 +92,7 @@
   # no_log: true
   lineinfile:
     state: present
-    line: "- {{ ansible_user }}@{{addr.public_ip_address}}"
+    line: "- {{ node_user }}@{{addr.public_ip_address}}"
     insertafter: EOF
     dest: "{{ workspace }}{{nodeLabel}}-sshhosts"
     create: true

From 5dd1b4220d66814d14d6f83d75970400a00c33c3 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Jan 2023 16:20:14 +0000
Subject: [PATCH 78/78] test deploy test infra

---
 .ci/ansible/tasks/runners.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/ansible/tasks/runners.yml b/.ci/ansible/tasks/runners.yml
index e5e15db75c..241b9a1af2 100644
--- a/.ci/ansible/tasks/runners.yml
+++ b/.ci/ansible/tasks/runners.yml
@@ -92,7 +92,7 @@
   # no_log: true
   lineinfile:
     state: present
-    line: "- {{ node_user }}@{{addr.public_ip_address}}"
+    line: "- {{ nodeUser }}@{{addr.public_ip_address}}"
     insertafter: EOF
     dest: "{{ workspace }}{{nodeLabel}}-sshhosts"
     create: true