Actualize separate build node deployment

playbooks/albs_with_separate_build_node.yml

@@ -11,7 +11,6 @@
   hosts: build_node_vm
   roles:
     - separate_build_node
-    - { role: ezamriy.fail2ban, fail2ban_ignoreip: '127.0.0.1/8 192.168.0.0/24' }
   tags:
     - build-node-deploy
   connection: "{{ 'local' if use_local_connection else 'ssh' }}"

roles/separate_build_node/defaults/main/common.yml

@@ -4,21 +4,15 @@ base_work_dir: "/srv/alternatives"
 home_dir: "/home/{{ service_user }}"
 base_conf_dir: "{{ home_dir }}/.config"
 final_conf_dir: "{{ base_conf_dir }}/castor"
+service_user: albs-builder
+service_group: albs-builder
 build_node_working_directory: "{{ home_dir }}/albs"
+build_node_venv_directory: "{{ home_dir }}/.builder-venv"
+build_node_requirements_path: "{{ build_node_working_directory }}/albs-node/requirements.txt"
+powertools_repository_name: crb
 working_directories:
   - "{{ base_work_dir }}"
   - "{{ build_node_working_directory }}"
-  - "/var/cache/pbuilder/aptcache/"
-  - "/var/cache/pbuilder/pbuilder_envs/"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/buster-amd64/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/bionic-amd64/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/focal-amd64/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/jessie-amd64/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/stretch-amd64/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/xenial-amd64/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/buster-arm64/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/buster-armhf/aptcache"
-  - "{{ base_work_dir }}/castor/build_node/pbuilder_envs/raspbian-armhf/aptcache"
   - "{{ base_work_dir }}/castor/build_node"
   - "{{ base_work_dir }}/castor/build_node/mock_configs"
   - "{{ final_conf_dir }}"

roles/separate_build_node/defaults/main/dnf.yml

@@ -1,35 +1,32 @@
 ---
-os_version: "8"
+os_version: "9"
 build_node_dnf_packages:
-  - "python3"
+  - "centpkg"
+  - "cmake"
+  - "cpio"
+  - "createrepo_c"
+  - "ef2sprogs"
+  - "fedpkg"
   - "gcc"
   - "gcc-c++"
-  - "python3-devel"
-  - "python3-virtualenv"
-  - "cmake"
-  - "python3-pycurl"
+  - "git"
+  - "htop"
+  - "kernel-rpm-macros"
+  - "keyrings-filesystem"
   - "libicu"
   - "libicu-devel"
-  - "python3-lxml"
-  - "git"
-  - "tree"
-  - "mlocate"
   - "mc"
-  - "createrepo_c"
+  - "mock"
+  - "mock-rpmautospec"
+  - "python3"
   - "python3-createrepo_c"
-  - "xmlsec1-openssl-devel"
-  - "cpio"
-  - "sudo"
-  - "kernel-rpm-macros"
+  - "python3-devel"
   - "python3-libmodulemd"
-  - "dpkg-dev"
-  - "mock"
-  - "debootstrap"
-  - "pbuilder"
-  - "apt"
-  - "apt-libs"
-  - "python3-apt"
-  - "keyrings-filesystem"
+  - "python3-lxml"
+  - "python3-pycurl"
+  - "python3-virtualenv"
+  - "sudo"
+  - "tree"
   - "ubu-keyring"
-  - "debian-keyring"
+  - "xmlsec1-openssl-devel"
 ...

roles/separate_build_node/tasks/common.yml

@@ -1,10 +1,15 @@
 ---
 
 - name: Allow 'wheel' group to have passwordless sudo
-  lineinfile:
+  ansible.builtin.lineinfile:
     dest: /etc/sudoers
     state: present
     regexp: '^%wheel'
     line: '%wheel ALL=(ALL) NOPASSWD: ALL'
     validate: 'visudo -cf %s'
+
+- name: Set SELinux mode to permissive
+  ansible.posix.selinux:
+    policy: targeted
+    state: permissive
 ...

roles/separate_build_node/tasks/create_env.yml

@@ -1,7 +1,7 @@
 ---
 
-- name: Create service user
-  user:
+- name: Create user "{{ service_user }}"
+  ansible.builtin.user:
     name: "{{ service_user }}"
     groups: wheel, mock
     append: yes
@@ -10,7 +10,7 @@
     generate_ssh_key: yes
 
 - name: Create subsequent directories
-  file:
+  ansible.builtin.file:
     path: "{{ item }}"
     state: directory
     recurse: yes
@@ -19,27 +19,40 @@
   with_items: "{{ working_directories }}"
 
 - name: Generate build node config
-  template:
+  ansible.builtin.template:
     src: build_node.j2
     dest: "{{ final_conf_dir }}/build_node.yml"
     owner: "{{ service_user }}"
     group: "{{ service_group }}"
     mode: "0644"
 
-- name: Cloud build node repository
-  git:
+- name: Clone albs-node repository
+  become: yes
+  become_user: "{{ service_user }}"
+  ansible.builtin.git:
     repo: https://github.com/AlmaLinux/albs-node.git
     dest: "{{ build_node_working_directory }}/albs-node"
     clone: yes
     update: yes
     accept_hostkey: yes
     force: yes
 
-- name: Create venv
+- name: Create Python virtual environment
   become: yes
   become_user: "{{ service_user }}"
-  pip:
-    virtualenv: "{{ build_node_venv_directory }}"
-    virtualenv_command: "python3 -m venv"
+  ansible.builtin.command: python3 -m venv --system-site-packages "{{ build_node_venv_directory }}"
+  args:
+    creates: "{{ build_node_venv_directory }}/bin/activate"
+
+- name: Upgrade pip in virtualenv
+  become: yes
+  become_user: "{{ service_user }}"
+  ansible.builtin.command: "{{ build_node_venv_directory }}/bin/pip install --upgrade pip"
+
+- name: Install requirements into virtualenv
+  become: yes
+  become_user: "{{ service_user }}"
+  ansible.builtin.pip:
     requirements: "{{ build_node_requirements_path }}"
-    virtualenv_site_packages: yes
+    virtualenv: "{{ build_node_venv_directory }}"
+...

roles/separate_build_node/tasks/dnf.yml

@@ -1,37 +1,37 @@
 ---
 
 - name: Update system packages
-  dnf:
+  ansible.builtin.dnf:
     name: "*"
     state: latest
   register: update_state
 
 - name: Reboot a machine
-  reboot:
+  ansible.builtin.reboot:
     reboot_timeout: 180
   when: update_state.changed
 
 - name: Install epel repository
-  dnf:
-    name: "epel-release"
+  ansible.builtin.dnf:
+    name: epel-release
     state: latest
 
 - name: Install build system repository
-  yum_repository:
+  ansible.builtin.yum_repository:
     name: "buildnode"
     file: "buildnode"
     description: "Repository with dependencies for albs-node code"
     baseurl: https://repo.almalinux.org/build_system/{{ os_version }}/$basearch/
     enabled: no
     gpgcheck: no
 
-- name: Install DNF packages
-  dnf:
+- name: Install required system packages packages
+  ansible.builtin.dnf:
     name: "{{ build_node_dnf_packages }}"
     state: latest
     enablerepo:
-      - "epel"
-      - "buildnode"
+      - epel
+      - buildnode
       - "{{ powertools_repository_name }}"
     update_cache: yes
 ...

roles/separate_build_node/tasks/install_systemd_service.yml

@@ -1,15 +1,15 @@
 ---
 - name: Create build node systemd service
-  template:
-    dest: "/etc/systemd/system/albs-build-node.service"
-    src: "albs_build_node.service.j2"
-    group: "root"
-    owner: "root"
-    mode: "0644"
+  ansible.builtin.template:
+    dest: /etc/systemd/system/albs-build-node.service
+    src: albs_build_node.service.j2
+    group: root
+    owner: root
+    mode: 0644
 
 - name: Enable and start build node systemd service
-  systemd:
-    name: "albs-build-node.service"
+  ansible.builtin.systemd:
+    name: albs-build-node.service
     masked: no
     state: restarted
     daemon_reload: yes

roles/separate_build_node/templates/albs_build_node.service.j2

@@ -1,5 +1,6 @@
 [Unit]
 Description=ALBS build node
+After=network.target remote-fs.target
 
 [Service]
 RemainAfterExit=no
@@ -8,7 +9,7 @@ RestartSec=5s
 Type=simple
 User={{ service_user }}
 Group={{ service_group }}
-ExecStart={{ build_node_venv_directory }}/bin/python3 {{ build_node_working_directory }}/albs-node/almalinux_build_node.py
+ExecStart=/bin/bash -c "{{ build_node_venv_directory }}/bin/python3 {{ build_node_working_directory }}/albs-node/almalinux_build_node.py"
 
 [Install]
 WantedBy=multi-user.target

roles/separate_build_node/templates/build_node.j2

@@ -1,11 +1,25 @@
 ---
 master_url: "{{ albs_api_url }}"
-base_arch: "{{ machine_arch }}"
 threads_count: {{ threads_count | default(4) }}
+base_arch: "{{ machine_arch }}"
+build_src: {{ build_src | default(True) }}
+{% if exclusions_url is defined and exclusions_url %}
+exclusions_url: "{{ exclusions_url }}"
+{% endif %}
 jwt_token: "{{ albs_jwt_token }}"
 pulp_host: "{{ pulp_host }}"
 pulp_user: "{{ pulp_user }}"
 pulp_password: "{{ pulp_password }}"
+pulp_uploader_max_workers: {{ pulp_uploader_workers | default(2) }}
+{% if sentry_dsn is defined and sentry_dsn %}
+sentry_dsn: "{{ sentry_dsn }}"
+{% endif %}
+{% if sentry_traces_sample_rate is defined and sentry_traces_sample_rate %}
+sentry_traces_sample_rate: {{ sentry_traces_sample_rate }}
+{% endif %}
+{% if sentry_environment is defined and sentry_environment %}
+sentry_environment: {{ sentry_environment }}
+{% endif %}
 {% if immudb_address is defined and immudb_address %}
 immudb_address: "{{ immudb_address }}"
 {% endif %}
@@ -17,4 +31,4 @@ immudb_username: "{{ immudb_username }}"
 {% endif %}
 {% if immudb_password is defined and immudb_password %}
 immudb_password: "{{ immudb_password }}"
-{% endif %}
+{% endif %}