From 1198db0f0831ca150b7be0a9b8b8a0b459f02186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20M=C3=A9lotte?= Date: Fri, 26 Jun 2020 15:59:55 +0200 Subject: [PATCH 1/6] docker: add a Dockerfile containing boardfarm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To run the boardfarm tests, we need an image containing boardfarm. Add an image that contains all the boardfarm dependencies, fetch a specific commit (the latest one from the master branch) and install it. The dockerfile was adapted from the existing work on: https://github.com/prplfoundation/prplMesh/pull/1483 Also add the image to .gitlab-ci to get it built automatically. Signed-off-by: Raphaël Mélotte --- .gitlab-ci.yml | 3 ++ tools/docker/boardfarm-ci/Dockerfile | 41 ++++++++++++++++++ tools/docker/boardfarm-ci/requirements.txt | 48 ++++++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 tools/docker/boardfarm-ci/Dockerfile create mode 100644 tools/docker/boardfarm-ci/requirements.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c64ba84133..51597517dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,9 @@ image-build-builder/ubuntu/bionic: image-build-tests-runner: extends: .image-build +image-build-boardfarm-ci: + extends: .image-build + .in-prplmesh-builder: image: name: $CI_REGISTRY_IMAGE/prplmesh-builder-ubuntu-bionic:$CI_PIPELINE_ID diff --git a/tools/docker/boardfarm-ci/Dockerfile b/tools/docker/boardfarm-ci/Dockerfile new file mode 100644 index 0000000000..a7e1bc06c4 --- /dev/null +++ b/tools/docker/boardfarm-ci/Dockerfile @@ -0,0 +1,41 @@ +FROM python:3.8-slim-buster + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates curl \ + curl \ + gcc \ + git \ + gnupg \ + gnupg-agent \ + libsnmp-dev \ + software-properties-common \ + wireshark-common \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/requirements.txt + +WORKDIR app +RUN pip3 install -r requirements.txt + +# TODO: what needs this? +#RUN pip3 install jsonschema distro + + +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -L "https://github.com/docker/compose/releases/download/1.26.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +RUN chmod 755 /usr/local/bin/docker-compose + +RUN git clone https://github.com/mattsm/boardfarm.git \ + && cd boardfarm \ + && git checkout 100521fde1fb67536682cafecc2f91a6e2e8a6f8 \ + && python3 setup.py install diff --git a/tools/docker/boardfarm-ci/requirements.txt b/tools/docker/boardfarm-ci/requirements.txt new file mode 100644 index 0000000000..aaaea868c8 --- /dev/null +++ b/tools/docker/boardfarm-ci/requirements.txt @@ -0,0 +1,48 @@ +aenum +argparse +beautifulsoup4 +boto3 +cdrouter +debtcollector +distro +dlipower +dnspython +docutils<0.16 +easysnmp +elasticsearch>=1.0.0 +Faker +future +ipaddress +jira +jsonschema +marshmallow<3 +matplotlib +nested_lookup==0.2.19 +netaddr +pexpect +pre-commit +psutil +pycountry +pycryptodome +pyflakes +pymongo +pyserial +pysmi +pysnmp +pytest==5.3.5 +pytest-html +pytest-mock +pytest-randomly +pyvirtualdisplay +requests +retry +selenium +simplejson +six +sphinx +termcolor +unittest2 +xmltodict +xvfbwrapper +yapf +zeep From 3fd1f05c42d3ee7afa38690892074451295e3d74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20M=C3=A9lotte?= Date: Fri, 26 Jun 2020 16:02:33 +0200 Subject: [PATCH 2/6] tests: boardfarm: use the installed version of boardfarm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To use an already-installed boardfarm version we don't need to know it's path, we can assume it's already in PATH. Signed-off-by: Raphaël Mélotte --- tests/run_bf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_bf.sh b/tests/run_bf.sh index 8c71953b2f..0dbbea4d12 100755 --- a/tests/run_bf.sh +++ b/tests/run_bf.sh @@ -15,4 +15,4 @@ else fi export PYTHONPATH -exec python3 "${bf_dir}"/bft -c "${bf_plugins_dir}"/boardfarm_prplmesh/prplmesh_config.json -n prplmesh_docker -x test_flows +exec bft -c "${bf_plugins_dir}"/boardfarm_prplmesh/prplmesh_config.json -n prplmesh_docker -x test_flows From 74041ac757b4a147fc1a7ceded2b838f80d2d07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20M=C3=A9lotte?= Date: Fri, 26 Jun 2020 16:04:36 +0200 Subject: [PATCH 3/6] ci: add a job to run the boardfarm tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because prplMesh is not currently relocatable, we have to run it both in the same directory it was built in, but also as the same user. It is currently built with the root user inside a docker container, with the USER variable set as ''. As a consequence, we have to run it as root as well, and explicitly set USER to an empty string (because boardfarm relies on it's presence). Since we want to run boardfarm in docker and boardfarm starts containers itself, we have to use a docker image where docker itself is available from within the container. For this 3 options were possible: - The current one, running another docker daemon inside the first docker container. It is a little bit overkill, and makes it hard to preserve the cache between runs. - Run with a Docker executor and bind-mount the Docker socket. It requires some care when we want to mount volumes from within a container, because the volumes have to be available on the host. - Run with a shell executor. This approach has the benefit of always having the prplMesh tree cloned and the artifacts available on the host. Since we currently run the containers as root however, the Gitlab runner won't be able to clean up files in-between runs. The option 2 was chosen, because it was the easiest to use at the moment. For now the job also needs only the docker builds, so add a "need" for it. Signed-off-by: Raphaël Mélotte --- .gitlab-ci.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51597517dc..a706e7ed2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -216,6 +216,21 @@ run-tests: - job: image-build-tests-runner - job: image-build-runner +boardfarm-tests: + stage: test + image: $CI_REGISTRY_IMAGE/prplmesh-boardfarm-ci:$CI_PIPELINE_ID + script: + - export USER='' + - tests/run_bf.sh + artifacts: + paths: + - logs + - results + needs: + - job: build-in-docker + tags: + - boardfarm + .build-for-openwrt: stage: build script: From 84d0507c63c4290fc61776273981804c6f2fe0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20M=C3=A9lotte?= Date: Mon, 29 Jun 2020 07:41:35 +0200 Subject: [PATCH 4/6] tests: boardfarm: error out and print the result when pexpect run fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a command failed to run, raise an exception early, and add the stdout and stderr output of the command in the message. These changes were taken and adapted from: https://github.com/prplfoundation/prplMesh/pull/1483 Signed-off-by: Raphaël Mélotte --- .../boardfarm_prplmesh/devices/prplmesh_base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_base.py b/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_base.py index 227e8cd27b..700c5c2863 100644 --- a/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_base.py +++ b/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_base.py @@ -8,14 +8,23 @@ from boardfarm.devices import linux +class CommandError(Exception): + """Raised on failed execution""" + pass + + class PrplMeshBase(linux.LinuxDevice): """PrplMesh abstract device.""" def _run_shell_cmd(self, cmd: str = "", args: list = None, timeout: int = 30): """Wrapper that executes command with specified args on host machine and logs output.""" - res = pexpect.run(cmd, args=args, timeout=timeout, encoding="utf-8") + res, exitstatus = pexpect.run(cmd, args=args, timeout=timeout, encoding="utf-8", + withexitstatus=1) entry = " ".join((cmd, " ".join(args))) + if exitstatus != 0: + raise CommandError("Error executing {}:\n{}".format(entry, res)) + self.log_calls += entry self.log += "$ " + entry + "\r\n" + res From 9dbd5febdca2c0d098427d2fde332ebdd3037d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20M=C3=A9lotte?= Date: Mon, 6 Jul 2020 16:31:40 +0200 Subject: [PATCH 5/6] tests: boardfarm: use docker_name for check_status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With https://github.com/prplfoundation/prplMesh/pull/1486, "name" was changed to "docker_name". The name in "check_status" was not updated, so do it now. It fixes the following error: Error: No such container: controller See for example: https://gitlab.com/prpl-foundation/prplMesh/-/jobs/625370585 Signed-off-by: Raphaël Mélotte --- .../boardfarm_prplmesh/devices/prplmesh_docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py b/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py index ae46897104..17aea4407b 100644 --- a/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py +++ b/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py @@ -78,7 +78,7 @@ def check_status(self): and also after test - to insure that device still operational. """ self._run_shell_cmd(os.path.join(rootdir, "tools", "docker", "test.sh"), - ["-v", "-n", self.name]) + ["-v", "-n", self.docker_name]) def isalive(self): """Method required by boardfarm. From 5f47678110ebecf47e7efbaefb3cc0417b3e47bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20M=C3=A9lotte?= Date: Mon, 6 Jul 2020 19:18:22 +0200 Subject: [PATCH 6/6] tests: boardfarm: use device_get_info for check_status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With https://github.com/prplfoundation/prplMesh/pull/1486, the first device to start has been changed to be the agent. Since the agent cannot be operational unless there is a controller already running on the same network, something else has to be used for "check_status". For now, use netcat to check that "device_get_info" works. Signed-off-by: Raphaël Mélotte --- .../boardfarm_prplmesh/devices/prplmesh_docker.py | 4 ++-- tools/docker/boardfarm-ci/Dockerfile | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py b/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py index 17aea4407b..7e890c800d 100644 --- a/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py +++ b/tests/boardfarm_plugins/boardfarm_prplmesh/devices/prplmesh_docker.py @@ -77,8 +77,8 @@ def check_status(self): It is used by boardfarm to indicate that spawned device instance is ready for test and also after test - to insure that device still operational. """ - self._run_shell_cmd(os.path.join(rootdir, "tools", "docker", "test.sh"), - ["-v", "-n", self.docker_name]) + self._run_shell_cmd("printf", + ["device_get_info", "|", "nc", "-w", "1", "controller-rme", "8002"]) def isalive(self): """Method required by boardfarm. diff --git a/tools/docker/boardfarm-ci/Dockerfile b/tools/docker/boardfarm-ci/Dockerfile index a7e1bc06c4..67fd3ed294 100644 --- a/tools/docker/boardfarm-ci/Dockerfile +++ b/tools/docker/boardfarm-ci/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update \ gnupg \ gnupg-agent \ libsnmp-dev \ + netcat \ software-properties-common \ wireshark-common \ && rm -rf /var/lib/apt/lists/*