diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7620862b61254..292ba4cb05e7b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -186,6 +186,31 @@ jobs: - name: Export NuttX Repo SHA run: echo "nuttx_sha=`git -C sources/nuttx rev-parse HEAD`" >> $GITHUB_ENV + - name: Install NTFC + uses: ./sources/nuttx/.github/actions/ci-container + env: + BLOBDIR: /tools/blobs + with: + run: | + # install venv + apt-get update + apt install -y python3-dev + apt install -y python3-venv + + # get NTFC sources + git clone -b release-0.0.1 https://github.com/szafonimateusz-mi/nuttx-ntfc + cd /github/workspace/nuttx-ntfc + + # install NTFC with venv + python3 -m venv /github/workspace/nuttx-ntfc/venv + source /github/workspace/nuttx-ntfc/venv/bin/activate + pip3 install . + deactivate + + # get NTFC test cases + cd external + git clone -b release-0.0.1 https://github.com/szafonimateusz-mi/nuttx-testing + - name: Run builds uses: ./sources/nuttx/.github/actions/ci-container env: @@ -194,6 +219,7 @@ jobs: run: | echo "::add-matcher::sources/nuttx/.github/gcc.json" export ARTIFACTDIR=`pwd`/buildartifacts + export NTFCDIR=/github/workspace/nuttx-ntfc git config --global --add safe.directory /github/workspace/sources/nuttx git config --global --add safe.directory /github/workspace/sources/apps cd sources/nuttx/tools/ci diff --git a/Documentation/guides/index.rst b/Documentation/guides/index.rst index 95457185b80ec..f1e11bc9a456b 100644 --- a/Documentation/guides/index.rst +++ b/Documentation/guides/index.rst @@ -16,7 +16,6 @@ Guides pysimcoder.rst customboards.rst customapps.rst - citests.rst zerolatencyinterrupts.rst fortify.rst nestedinterrupts.rst diff --git a/Documentation/index.rst b/Documentation/index.rst index 1443b326365f8..dc7773ccb6e6e 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -37,6 +37,7 @@ Last Updated: |today| reference/index.rst faq/index.rst debugging/index.rst + testing/index.rst guides/index.rst standards/index.rst glossary.rst diff --git a/Documentation/guides/citests.rst b/Documentation/testing/citests.rst similarity index 63% rename from Documentation/guides/citests.rst rename to Documentation/testing/citests.rst index 0d3b73fa8bcae..dbbffb0509672 100644 --- a/Documentation/guides/citests.rst +++ b/Documentation/testing/citests.rst @@ -7,8 +7,8 @@ Running CI Test Locally NuttX automatically runs continuous integration (CI) tests on `simulator `__ -target when new pull request is submitted. To avoid the tests failing you can -also run them locally on your computer prior to submitting new pull request. +and QEMU targets when a new pull request is submitted. To avoid the tests failing +you can also run them locally on your computer prior to submitting a new pull request. This page describes the step by step manual to do so. Configuring NuttX @@ -24,14 +24,6 @@ compiled followingly. $ ./tools/configure.sh sim:citest $ make -Now you can run the simulator to check the configuration was successful. - - .. code-block:: console - - $ ./nuttx - login: admin - password: Administrator - You should see NuttX shell with built in test applications. Now you can exit the simulator. @@ -41,8 +33,9 @@ the simulator. $ $ # we're back at the Linux prompt. -Running CI Tests -================ + +Running CI Tests with ``tools/ci/testrun`` +========================================== Running CI tests locally requires Minicom and Python 3.6 or newer to be installed on the system. Other requirements can be installed with following @@ -64,3 +57,22 @@ by following command. Where nuttx-path is an absolute path to NuttX root directory and log-path is a user defined directory to which tests log are saved. + +CI with NTFC +============ + +NTFC (NuttX Test Framework for Community) enables automated testing for NuttX +across simulator, QEMU, and real hardware via serial interface. The framework +automatically detects available applications in NuttX images and executes +applicable tests using pytest-based test cases. + +The framework and official test cases are available at: + +- https://github.com/szafonimateusz-mi/nuttx-ntfc +- https://github.com/szafonimateusz-mi/nuttx-testing + +Detailed documentation is available in the framework repository. + +CI migration to NTFC is in progress. Configurations currently using NTFC: + +- ``sim/citest`` diff --git a/Documentation/testing/index.rst b/Documentation/testing/index.rst new file mode 100644 index 0000000000000..d5824a9742619 --- /dev/null +++ b/Documentation/testing/index.rst @@ -0,0 +1,8 @@ +======= +Testing +======= + +This page contains information related to NuttX testing. + +.. toctree:: + citests.rst diff --git a/boards/arm/imx6/sabre-6quad/configs/citest/run b/boards/arm/imx6/sabre-6quad/configs/citest/run.sh similarity index 100% rename from boards/arm/imx6/sabre-6quad/configs/citest/run rename to boards/arm/imx6/sabre-6quad/configs/citest/run.sh diff --git a/boards/risc-v/qemu-rv/rv-virt/configs/citest/run b/boards/risc-v/qemu-rv/rv-virt/configs/citest/run.sh similarity index 100% rename from boards/risc-v/qemu-rv/rv-virt/configs/citest/run rename to boards/risc-v/qemu-rv/rv-virt/configs/citest/run.sh diff --git a/boards/risc-v/qemu-rv/rv-virt/configs/citest64/run b/boards/risc-v/qemu-rv/rv-virt/configs/citest64/run.sh similarity index 100% rename from boards/risc-v/qemu-rv/rv-virt/configs/citest64/run rename to boards/risc-v/qemu-rv/rv-virt/configs/citest64/run.sh diff --git a/boards/sim/sim/sim/configs/citest/config.yaml b/boards/sim/sim/sim/configs/citest/config.yaml new file mode 100644 index 0000000000000..84ee4c75599db --- /dev/null +++ b/boards/sim/sim/sim/configs/citest/config.yaml @@ -0,0 +1,12 @@ +config: + cwd: './' + timeout_session: 6000 + +product: + name: "ntfc-sim" + cores: + core0: + name: 'main' + device: 'sim' + conf_path: './.config' + elf_path: './nuttx' diff --git a/boards/sim/sim/sim/configs/citest/defconfig b/boards/sim/sim/sim/configs/citest/defconfig index b8202578eef45..6f371fb9a4698 100644 --- a/boards/sim/sim/sim/configs/citest/defconfig +++ b/boards/sim/sim/sim/configs/citest/defconfig @@ -131,6 +131,7 @@ CONFIG_START_YEAR=2008 CONFIG_SYSTEM_DUMPSTACK=y CONFIG_SYSTEM_NSH=y CONFIG_SYSTEM_POPEN=y +CONFIG_SYSTEM_SETLOGMASK=y CONFIG_TESTING_CMOCKA=y CONFIG_TESTING_CXXTEST=y CONFIG_TESTING_DRIVER_TEST=y diff --git a/boards/sim/sim/sim/configs/citest/run b/boards/sim/sim/sim/configs/citest/run deleted file mode 120000 index c9e283cbeb05c..0000000000000 --- a/boards/sim/sim/sim/configs/citest/run +++ /dev/null @@ -1 +0,0 @@ -../../../../../../tools/ci/cirun.sh \ No newline at end of file diff --git a/boards/sim/sim/sim/configs/citest/run.sh b/boards/sim/sim/sim/configs/citest/run.sh new file mode 100755 index 0000000000000..0fddb7b01bf10 --- /dev/null +++ b/boards/sim/sim/sim/configs/citest/run.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +############################################################################ +# boards/sim/sim/sim/configs/citest/run.sh +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +set -o xtrace + +# start from nuttx dir +olddir=$(pwd) +nuttdir=${CURRENTCONFDIR}/../../../../../../ +cd ${nuttdir} + +# enable venv +source ${NTFCDIR}/venv/bin/activate + +# run NTFC +confpath=${CURRENTCONFDIR}/config.yaml +jsonconf=${CURRENTCONFDIR}/session.json +testpath=${NTFCDIR}/external/nuttx-testing +python3 -m ntfc test --testpath=${testpath} --confpath=${confpath} --jsonconf=${jsonconf} + +ret="$?" +echo $ret + +# disable venv +deactivate + +# export test results +artifacts=${ARTIFACTCONFDIR}/ntfc +mkdir -p ${artifacts} +mv pytest.debug.log ${artifacts} +mv result ${artifacts} + +# restore old dir +cd ${olddir} + +exit $ret diff --git a/boards/sim/sim/sim/configs/citest/session.json b/boards/sim/sim/sim/configs/citest/session.json new file mode 100644 index 0000000000000..df9d9cbd86ef3 --- /dev/null +++ b/boards/sim/sim/sim/configs/citest/session.json @@ -0,0 +1,14 @@ +{ + "module": { + "include_module": [], + "exclude_module": + [ + "Nuttx_System_Driver_Pm", + "Nuttx_System_Fs_Fs" + ], + "order": [] + }, + "args": { + "kv": [] + } +} diff --git a/tools/testbuild.sh b/tools/testbuild.sh index bef77e35e2e83..0bafcc8981ae9 100755 --- a/tools/testbuild.sh +++ b/tools/testbuild.sh @@ -492,9 +492,11 @@ function refresh { function run { if [ ${RUN} -ne 0 ] && [ -z ${cmake} ]; then - run_script="$path/run" + run_script="$path/run.sh" if [ -x $run_script ]; then echo " Running NuttX..." + export ARTIFACTCONFDIR=$ARTIFACTDIR/$(echo $config | sed "s/:/\//")/ + export CURRENTCONFDIR=$(realpath $path) if ! $run_script; then fail=1 fi