initial support for automated testing with NTFC

.github/workflows/build.yml

@@ -186,6 +186,31 @@ jobs:
       - name: Export NuttX Repo SHA
         run: echo "nuttx_sha=`git -C sources/nuttx rev-parse HEAD`" >> $GITHUB_ENV
 
+      - name: Install NTFC
+        uses: ./sources/nuttx/.github/actions/ci-container
+        env:
+          BLOBDIR: /tools/blobs
+        with:
+          run: |
+            # install venv
+            apt-get update
+            apt install -y python3-dev
+            apt install -y python3-venv
+
+            # get NTFC sources
+            git clone -b release-0.0.1 https://github.com/szafonimateusz-mi/nuttx-ntfc
+            cd /github/workspace/nuttx-ntfc
+
+            # install NTFC with venv
+            python3 -m venv /github/workspace/nuttx-ntfc/venv
+            source /github/workspace/nuttx-ntfc/venv/bin/activate
+            pip3 install .
+            deactivate
+
+            # get NTFC test cases
+            cd external
+            git clone -b release-0.0.1 https://github.com/szafonimateusz-mi/nuttx-testing
+
       - name: Run builds
         uses: ./sources/nuttx/.github/actions/ci-container
         env:
@@ -194,6 +219,7 @@ jobs:
           run: |
             echo "::add-matcher::sources/nuttx/.github/gcc.json"
             export ARTIFACTDIR=`pwd`/buildartifacts
+            export NTFCDIR=/github/workspace/nuttx-ntfc
             git config --global --add safe.directory /github/workspace/sources/nuttx
             git config --global --add safe.directory /github/workspace/sources/apps
             cd sources/nuttx/tools/ci

Documentation/guides/index.rst

@@ -16,7 +16,6 @@ Guides
   pysimcoder.rst
   customboards.rst
   customapps.rst
-  citests.rst
   zerolatencyinterrupts.rst
   fortify.rst
   nestedinterrupts.rst

Documentation/index.rst

@@ -37,6 +37,7 @@ Last Updated: |today|
    reference/index.rst
    faq/index.rst
    debugging/index.rst
+   testing/index.rst
    guides/index.rst
    standards/index.rst
    glossary.rst

Documentation/guides/citests.rst → Documentation/testing/citests.rst

@@ -7,8 +7,8 @@ Running CI Test Locally
 
 NuttX automatically runs continuous integration (CI) tests on 
 `simulator <https://nuttx.apache.org/docs/latest/guides/simulator.html>`__
-target when new pull request is submitted. To avoid the tests failing you can
-also run them locally on your computer prior to submitting new pull request.
+and QEMU targets when a new pull request is submitted. To avoid the tests failing
+you can also run them locally on your computer prior to submitting a new pull request.
 This page describes the step by step manual to do so.
 
 Configuring NuttX
@@ -24,14 +24,6 @@ compiled followingly.
       $ ./tools/configure.sh sim:citest
       $ make
 
-Now you can run the simulator to check the configuration was successful.
-
-  .. code-block:: console
-
-      $ ./nuttx
-      login: admin
-      password: Administrator
-
 You should see NuttX shell with built in test applications. Now you can exit
 the simulator.
 
@@ -41,8 +33,9 @@ the simulator.
       $
       $ # we're back at the Linux prompt.
 
-Running CI Tests
-================
+
+Running CI Tests with ``tools/ci/testrun``
+==========================================
 
 Running CI tests locally requires Minicom and Python 3.6 or newer to be
 installed on the system. Other requirements can be installed with following
@@ -64,3 +57,22 @@ by following command.
 
 Where nuttx-path is an absolute path to NuttX root directory and log-path is
 a user defined directory to which tests log are saved.
+
+CI with NTFC
+============
+
+NTFC (NuttX Test Framework for Community) enables automated testing for NuttX
+across simulator, QEMU, and real hardware via serial interface. The framework
+automatically detects available applications in NuttX images and executes
+applicable tests using pytest-based test cases.
+
+The framework and official test cases are available at:
+
+- https://github.com/szafonimateusz-mi/nuttx-ntfc
+- https://github.com/szafonimateusz-mi/nuttx-testing
+
+Detailed documentation is available in the framework repository.
+
+CI migration to NTFC is in progress. Configurations currently using NTFC:
+
+- ``sim/citest``

Documentation/testing/index.rst

@@ -0,0 +1,8 @@
+=======
+Testing
+=======
+
+This page contains information related to NuttX testing.
+
+.. toctree::
+  citests.rst

boards/sim/sim/sim/configs/citest/config.yaml

@@ -0,0 +1,12 @@
+config:
+  cwd: './'
+  timeout_session: 6000
+
+product:
+  name: "ntfc-sim"
+  cores:
+    core0:
+      name: 'main'
+      device: 'sim'
+      conf_path: './.config'
+      elf_path: './nuttx'

boards/sim/sim/sim/configs/citest/defconfig

@@ -131,6 +131,7 @@ CONFIG_START_YEAR=2008
 CONFIG_SYSTEM_DUMPSTACK=y
 CONFIG_SYSTEM_NSH=y
 CONFIG_SYSTEM_POPEN=y
+CONFIG_SYSTEM_SETLOGMASK=y
 CONFIG_TESTING_CMOCKA=y
 CONFIG_TESTING_CXXTEST=y
 CONFIG_TESTING_DRIVER_TEST=y

boards/sim/sim/sim/configs/citest/run.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+############################################################################
+# boards/sim/sim/sim/configs/citest/run.sh
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.  The
+# ASF licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+############################################################################
+
+set -o xtrace
+
+# start from nuttx dir
+olddir=$(pwd)
+nuttdir=${CURRENTCONFDIR}/../../../../../../
+cd ${nuttdir}
+
+# enable venv
+source ${NTFCDIR}/venv/bin/activate
+
+# run NTFC
+confpath=${CURRENTCONFDIR}/config.yaml
+jsonconf=${CURRENTCONFDIR}/session.json
+testpath=${NTFCDIR}/external/nuttx-testing
+python3 -m ntfc test --testpath=${testpath} --confpath=${confpath} --jsonconf=${jsonconf}
+
+ret="$?"
+echo $ret
+
+# disable venv
+deactivate
+
+# export test results
+artifacts=${ARTIFACTCONFDIR}/ntfc
+mkdir -p ${artifacts}
+mv pytest.debug.log ${artifacts}
+mv result ${artifacts}
+
+# restore old dir
+cd ${olddir}
+
+exit $ret

boards/sim/sim/sim/configs/citest/session.json

@@ -0,0 +1,14 @@
+{
+    "module": {
+        "include_module": [],
+        "exclude_module":
+        [
+            "Nuttx_System_Driver_Pm",
+            "Nuttx_System_Fs_Fs"
+        ],
+        "order": []
+    },
+    "args": {
+        "kv": []
+    }
+}

tools/testbuild.sh

@@ -492,9 +492,11 @@ function refresh {
 
 function run {
   if [ ${RUN} -ne 0 ] && [ -z ${cmake} ]; then
-    run_script="$path/run"
+    run_script="$path/run.sh"
     if [ -x $run_script ]; then
       echo "  Running NuttX..."
+      export ARTIFACTCONFDIR=$ARTIFACTDIR/$(echo $config | sed "s/:/\//")/
+      export CURRENTCONFDIR=$(realpath $path)
       if ! $run_script; then
         fail=1
       fi