From d5cb8e090edff5abdc63946852d3782184066600 Mon Sep 17 00:00:00 2001 From: ravalsam Date: Thu, 18 Sep 2025 22:41:40 +0000 Subject: [PATCH 01/40] SRE-3150 Test: This is Debug PR for NVMe issue Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: ravalsam --- Jenkinsfile | 1 + src/tests/ftest/pool/create_all_hw.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 756892b901d..ccf7c836e8b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1161,6 +1161,7 @@ pipeline { nvme: 'auto_md_on_ssd', run_if_pr: true, run_if_landing: false, + image_version: 'el9.5', job_status: job_status_internal ), 'Functional Hardware Medium VMD': getFunctionalTestStage( diff --git a/src/tests/ftest/pool/create_all_hw.py b/src/tests/ftest/pool/create_all_hw.py index 66f0cec8080..8b166439639 100644 --- a/src/tests/ftest/pool/create_all_hw.py +++ b/src/tests/ftest/pool/create_all_hw.py @@ -85,7 +85,7 @@ def test_recycle_pools_hw(self): pool_count = self.params.get("pool_count", "/run/test_recycle_pools_hw/*", 0) deltas_bytes = self.get_deltas("test_recycle_pools_hw") deltas_bytes[:] = [it * self.engines_count for it in deltas_bytes] - self.log.info("Test pool creation and destruction") + self.log.info("Test pool creation and destruction -- TESTING SAMIR --") self.log.info("\t- pool_count=%d", pool_count) self.log_deltas(*deltas_bytes) From 6c78311ef5b4e845ebc1aae293cc05a1394dd87c Mon Sep 17 00:00:00 2001 From: ravalsam Date: Mon, 29 Sep 2025 15:35:13 +0000 Subject: [PATCH 02/40] Reverting it back to original OS. Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: ravalsam --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ccf7c836e8b..756892b901d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1161,7 +1161,6 @@ pipeline { nvme: 'auto_md_on_ssd', run_if_pr: true, run_if_landing: false, - image_version: 'el9.5', job_status: job_status_internal ), 'Functional Hardware Medium VMD': getFunctionalTestStage( From fdb99d23309fea174b52423e54299dfdbfefb935 Mon Sep 17 00:00:00 2001 From: ravalsam Date: Mon, 27 Oct 2025 18:04:35 +0000 Subject: [PATCH 03/40] Comment node prep NVMe check. Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: ravalsam --- ci/functional/test_main_prep_node.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/functional/test_main_prep_node.sh b/ci/functional/test_main_prep_node.sh index fec054efb54..43c847c13e5 100755 --- a/ci/functional/test_main_prep_node.sh +++ b/ci/functional/test_main_prep_node.sh @@ -242,7 +242,7 @@ if [ "$ib_count" -ge 2 ]; then echo "OK: Found $dimm_rcount DIMM PMEM regions." fi testcases+=" $nl" - +<${nl}" if [ "$lsblk_pmem" -ne "$dimm_rcount" ]; then From 9f60be93521b2c6e25e2036ddbe23da5b7d7cab1 Mon Sep 17 00:00:00 2001 From: Samir Raval Date: Thu, 13 Nov 2025 10:03:49 -0800 Subject: [PATCH 04/40] Update test_main_prep_node.sh Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: ravalsam Signed-off-by: Samir Raval --- ci/functional/test_main_prep_node.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/functional/test_main_prep_node.sh b/ci/functional/test_main_prep_node.sh index 43c847c13e5..fec054efb54 100755 --- a/ci/functional/test_main_prep_node.sh +++ b/ci/functional/test_main_prep_node.sh @@ -242,7 +242,7 @@ if [ "$ib_count" -ge 2 ]; then echo "OK: Found $dimm_rcount DIMM PMEM regions." fi testcases+=" $nl" -<${nl}" if [ "$lsblk_pmem" -ne "$dimm_rcount" ]; then From e4a7c38c3fa6f20b29d0ed0350d4132a9b7cc2b1 Mon Sep 17 00:00:00 2001 From: ravalsam Date: Fri, 14 Nov 2025 01:21:06 +0000 Subject: [PATCH 05/40] Testing changes. Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: ravalsam --- src/tests/ftest/util/pool_create_all_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/util/pool_create_all_base.py b/src/tests/ftest/util/pool_create_all_base.py index 660626c2c8e..f0f05f50ef4 100644 --- a/src/tests/ftest/util/pool_create_all_base.py +++ b/src/tests/ftest/util/pool_create_all_base.py @@ -226,7 +226,7 @@ def check_pool_recycling(self, pool_count, scm_delta_bytes, nvme_delta_bytes=Non first_pool_size = None for index in range(pool_count): self.log.info("Creating pool %d with all the available storage: size=100%%", index) - self.pool[index].size.update("100%", "pool[0].size") + self.pool[index].size.update("90%", "pool[0].size") self.pool[index].create() self.pool[index].get_info() s_total = self.pool[index].info.pi_space.ps_space.s_total From 3f3b111c38fa2e68cd7f9ce79105626d3f3fe418 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 25 Nov 2025 09:30:42 +0100 Subject: [PATCH 06/40] Break tests repetition after first failure Skip-build-el9-gcc: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- src/tests/ftest/util/launch_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index cd8bf8eeed2..d992072c348 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1296,6 +1296,10 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop if not code_coverage.setup(logger, result.tests[0]): return_code |= 128 + # ignore return code from coverage setup as tests can still run until first failure + if repeat > 1: + return_code = 0 + self._details["tests"] = [] # Run each test for as many repetitions as requested @@ -1345,6 +1349,10 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop # Stop logging to the test log file logger.removeHandler(test_file_handler) + if repeat > 1 and return_code != 0: + logger.info("Failure at test repetition %s/%s. No more repetitions.", loop, repeat) + break + # Cleanup any specified files at the end of testing for file, info in cleanup_files.items(): command = command_as_user(f"rm -fr {file}", info['user']) From aecf0b348200701e4d81e4819584d04a9e05c9ae Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 25 Nov 2025 21:27:39 +0100 Subject: [PATCH 07/40] Default build with pre-set configuration Cancel-prev-build: false Skip-build-el9-gcc: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 756892b901d..dc5e03091f1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -252,7 +252,7 @@ pipeline { description: 'Type of build. Passed to scons as BUILD_TYPE. (I.e. dev, release, debug, etc.). ' + 'Defaults to release on an RC or dev otherwise.') string(name: 'TestRepeat', - defaultValue: '', + defaultValue: '10', description: 'Test-repeat to use for this run. Specifies the ' + 'number of times to repeat each functional test. ' + 'CAUTION: only use in combination with a reduced ' + @@ -389,7 +389,7 @@ pipeline { defaultValue: 'ci_nlt_1', description: 'Label to use for NLT tests') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_LABEL', - defaultValue: 'ci_nvme5', + defaultValue: 'ci_node-hdr-210_212-219X', description: 'Label to use for the Functional Hardware Medium (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL', defaultValue: 'ci_ofed5', From 19a6a3c763f0a981c2efea3347e601972c9cacff Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 27 Nov 2025 15:48:08 +0100 Subject: [PATCH 08/40] Sanity check on el9.5 Cancel-prev-build: false Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc5e03091f1..d3fd5f16aaf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -252,7 +252,7 @@ pipeline { description: 'Type of build. Passed to scons as BUILD_TYPE. (I.e. dev, release, debug, etc.). ' + 'Defaults to release on an RC or dev otherwise.') string(name: 'TestRepeat', - defaultValue: '10', + defaultValue: '20', description: 'Test-repeat to use for this run. Specifies the ' + 'number of times to repeat each functional test. ' + 'CAUTION: only use in combination with a reduced ' + @@ -1159,6 +1159,7 @@ pipeline { stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', + image_version: 'el9.5', run_if_pr: true, run_if_landing: false, job_status: job_status_internal From 8aed634c2c3b1434f4da9110eabdfd8caa41526b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 28 Nov 2025 20:02:50 +0100 Subject: [PATCH 09/40] Skip leap 15 build Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d3fd5f16aaf..d554930ded5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -303,7 +303,7 @@ pipeline { defaultValue: false, description: 'Do not build sources and RPMs on EL 9') booleanParam(name: 'CI_leap15_NOBUILD', - defaultValue: false, + defaultValue: true, description: 'Do not build sources and RPMs on Leap 15') booleanParam(name: 'CI_ALLOW_UNSTABLE_TEST', defaultValue: false, From 481de859ab5867df71da3f3d0858f0440136ce5f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 28 Nov 2025 20:04:30 +0100 Subject: [PATCH 10/40] Revert "Sanity check on el9.5" This reverts commit e3da5b4b47978dad28b320fe6d1251c919254b3e. Cancel-prev-build: false Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d554930ded5..31cac06b8b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -252,7 +252,7 @@ pipeline { description: 'Type of build. Passed to scons as BUILD_TYPE. (I.e. dev, release, debug, etc.). ' + 'Defaults to release on an RC or dev otherwise.') string(name: 'TestRepeat', - defaultValue: '20', + defaultValue: '10', description: 'Test-repeat to use for this run. Specifies the ' + 'number of times to repeat each functional test. ' + 'CAUTION: only use in combination with a reduced ' + @@ -1159,7 +1159,6 @@ pipeline { stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', - image_version: 'el9.5', run_if_pr: true, run_if_landing: false, job_status: job_status_internal From 48bc3144e8350d23389bb7a6653cfbf2d50f2f7b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 8 Dec 2025 21:01:55 +0100 Subject: [PATCH 11/40] Let's try 9.5 again Signed-off-by: Tomasz Gromadzki Cancel-prev-build: false Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw --- Jenkinsfile | 3 ++- src/tests/ftest/util/launch_utils.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 31cac06b8b1..307c3762d95 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -389,7 +389,7 @@ pipeline { defaultValue: 'ci_nlt_1', description: 'Label to use for NLT tests') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_LABEL', - defaultValue: 'ci_node-hdr-210_212-219X', + defaultValue: 'ci_node-hdr-200_202-205X', description: 'Label to use for the Functional Hardware Medium (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL', defaultValue: 'ci_ofed5', @@ -1159,6 +1159,7 @@ pipeline { stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', + image_version: 'el9.5', run_if_pr: true, run_if_landing: false, job_status: job_status_internal diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index d992072c348..c0c88da2d70 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1287,6 +1287,7 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop int: status code indicating any issues running tests """ return_code = 0 + return_execute_code = 0 runner = TestRunner(self._avocado, result, len(self.tests), repeat, self.tag_filters) # Display the location of the avocado logs @@ -1336,7 +1337,9 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop continue # Run the test with avocado - return_code |= runner.execute(logger, test, loop, sequence + 1, sparse, fail_fast) + return_execute_code = runner.execute(logger, test, loop, sequence + 1, sparse, + fail_fast) + return_code |= return_execute_code # Archive the test results return_code |= runner.process( @@ -1349,8 +1352,9 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop # Stop logging to the test log file logger.removeHandler(test_file_handler) - if repeat > 1 and return_code != 0: - logger.info("Failure at test repetition %s/%s. No more repetitions.", loop, repeat) + if repeat > 1 and return_execute_code != 0: + logger.info("Failure at test repetition %s/%s: %d. ", loop, repeat, + return_execute_code) break # Cleanup any specified files at the end of testing From 74ebd21d0a46b7c0eda8c44a6dd922431a61be29 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 9 Dec 2025 08:19:01 +0100 Subject: [PATCH 12/40] Add parameter to controll HW stages image version Signed-off-by: Tomasz Gromadzki Cancel-prev-build: false Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw --- Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 307c3762d95..3b8a9451bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -403,6 +403,9 @@ pipeline { string(name: 'FUNCTIONAL_HARDWARE_LARGE_LABEL', defaultValue: 'ci_nvme9', description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') + string(name: 'FUNCTIONAL_HARDWARE_IMAGE_VERSION', + defaultValue: 'el8.8', + description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') string(name: 'CI_STORAGE_PREP_LABEL', defaultValue: '', description: 'Label for cluster to do a DAOS Storage Preparation') @@ -1159,7 +1162,7 @@ pipeline { stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', - image_version: 'el9.5', + image_version: params.FUNCTIONAL_HARDWARE_IMAGE_VERSION, run_if_pr: true, run_if_landing: false, job_status: job_status_internal From 455f4d52e75ad6370a862d3ce82102e6aa4380a9 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 9 Dec 2025 22:36:23 +0100 Subject: [PATCH 13/40] Stop tests as soon as fault is detected Cancel-prev-build: false Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 10 ++++++---- src/tests/ftest/util/launch_utils.py | 4 ++++ utils/tools/reset-nvme-namespace.sh | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 utils/tools/reset-nvme-namespace.sh diff --git a/Jenkinsfile b/Jenkinsfile index 3b8a9451bb9..e4f206083bf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -389,7 +389,8 @@ pipeline { defaultValue: 'ci_nlt_1', description: 'Label to use for NLT tests') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_LABEL', - defaultValue: 'ci_node-hdr-200_202-205X', + // defaultValue: 'ci_node-hdr-200_202-205X', + defaultValue: 'ci_node-hdr-210_212-215X', description: 'Label to use for the Functional Hardware Medium (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL', defaultValue: 'ci_ofed5', @@ -403,9 +404,9 @@ pipeline { string(name: 'FUNCTIONAL_HARDWARE_LARGE_LABEL', defaultValue: 'ci_nvme9', description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') - string(name: 'FUNCTIONAL_HARDWARE_IMAGE_VERSION', + string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION', defaultValue: 'el8.8', - description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') + description: 'Label to use for 5 node Functional Hardware Medium (MD on SSD) stages') string(name: 'CI_STORAGE_PREP_LABEL', defaultValue: '', description: 'Label for cluster to do a DAOS Storage Preparation') @@ -1150,6 +1151,7 @@ pipeline { stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto', + image_version: params.FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION, run_if_pr: false, run_if_landing: false, job_status: job_status_internal @@ -1162,7 +1164,7 @@ pipeline { stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', - image_version: params.FUNCTIONAL_HARDWARE_IMAGE_VERSION, + image_version: params.FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION, run_if_pr: true, run_if_landing: false, job_status: job_status_internal diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index c0c88da2d70..f7477027358 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1351,6 +1351,10 @@ def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop # Stop logging to the test log file logger.removeHandler(test_file_handler) + if repeat > 1 and return_execute_code != 0: + logger.info("Failure at test repetition %s/%s/%d: %d. ", loop, repeat, + sequence + 1, return_execute_code) + break if repeat > 1 and return_execute_code != 0: logger.info("Failure at test repetition %s/%s: %d. ", loop, repeat, diff --git a/utils/tools/reset-nvme-namespace.sh b/utils/tools/reset-nvme-namespace.sh new file mode 100644 index 00000000000..d15836c0318 --- /dev/null +++ b/utils/tools/reset-nvme-namespace.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e + +nvme delete-ns /dev/nvme0 -n 0x1 +nvme reset /dev/nvme0 +nvme create-ns /dev/nvme0 --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 +nvme attach-ns /dev/nvme0 -n 0x1 -c 0x41 +nvme reset /dev/nvme0 + +nvme delete-ns /dev/nvme1 -n 0x1 +nvme reset /dev/nvme1 +nvme create-ns /dev/nvme1 --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 +nvme attach-ns /dev/nvme1 -n 0x1 -c 0x41 +nvme reset /dev/nvme1 + +daos_server nvme scan \ No newline at end of file From fde1309dbcc5734244bc885ae3626f565a74bc0a Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 10 Dec 2025 21:08:39 +0000 Subject: [PATCH 14/40] nvme drives setup script Doc-only: true Signed-off-by: Tomasz Gromadzki --- utils/tools/reset-nvme-namespace.sh | 15 ---- utils/tools/unbind_bind_nvme.sh | 112 ++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 15 deletions(-) delete mode 100644 utils/tools/reset-nvme-namespace.sh create mode 100644 utils/tools/unbind_bind_nvme.sh diff --git a/utils/tools/reset-nvme-namespace.sh b/utils/tools/reset-nvme-namespace.sh deleted file mode 100644 index d15836c0318..00000000000 --- a/utils/tools/reset-nvme-namespace.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -e - -nvme delete-ns /dev/nvme0 -n 0x1 -nvme reset /dev/nvme0 -nvme create-ns /dev/nvme0 --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 -nvme attach-ns /dev/nvme0 -n 0x1 -c 0x41 -nvme reset /dev/nvme0 - -nvme delete-ns /dev/nvme1 -n 0x1 -nvme reset /dev/nvme1 -nvme create-ns /dev/nvme1 --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 -nvme attach-ns /dev/nvme1 -n 0x1 -c 0x41 -nvme reset /dev/nvme1 - -daos_server nvme scan \ No newline at end of file diff --git a/utils/tools/unbind_bind_nvme.sh b/utils/tools/unbind_bind_nvme.sh new file mode 100644 index 00000000000..97f273a3ade --- /dev/null +++ b/utils/tools/unbind_bind_nvme.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# +# Copyright 2025 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# Full reinitialization of all NVMe devices on the DAOS cluster node +# - unmount all NVMe devices +# - unbind all NVMe devices and re-bind them to the nvme driver in sorted PCI address order. +# - create new logical block device (namespace) on each NVMe drive +# - mount all except two logical devices to build configuration required for DAOS cluster node +# - initialize two logical devices to work with SPDK (via vfio-pci driver) +# - verify configuration using the `daos_server nvme scan` command. +# +# set -e +# set -x +set -u + +echo "Unmounting all /dev/nvme[0-15]n1 mountpoints (if any)..." + +for nvme in nvme{0..15}n1; do + dev="/dev/${nvme}" + mnt="/mnt/${nvme}" + if [ -b "$dev" ]; then + # Get mountpoint (empty if not mounted) + mp=$(lsblk -nr -o MOUNTPOINT "$dev") + if [ -n "$mp" ]; then + echo "Unmounting $dev from $mp" + sudo umount -f "$dev" || sudo umount -f "$mp" + else + echo "Umounting ${mnt}" + sudo umount -f "${mnt}" + fi + fi + if [ -d "${mnt}" ]; then + echo "Force umount of ${mnt}" + sudo umount -f "${mnt}" + fi +done + +# Find all PCI addresses for NVMe controllers +nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') + +if [ -z "$nvme_pcie_addrs" ]; then + echo "No NVMe PCI devices found." + exit 1 +fi + +echo "Found PCI NVMe addresses:" +echo "$nvme_pcie_addrs" +echo + +# Unbind all NVMe devices +echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." +for addr in $nvme_pcie_addrs; do + echo "Unbinding $addr" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind +done + +echo +echo "Binding NVMe devices to nvme driver in sorted order..." +for addr in $(echo "$nvme_pcie_addrs" | sort); do + echo "Binding $addr" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind +done + +# lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks +# lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) +# lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata +# lbaf 3 : ms:8 lbads:12 rp:0x2 → 4K + 8B metadata +# lbaf 4 : ms:64 lbads:12 rp:0x3 → 4K + 64B metadata +# lbads = log2(block size). +# 9 → 2⁹ = 512 bytes +# 12 → 2¹² = 4096 bytes (4K) +# ms = metadata size per block (0, 8, or 64 bytes). +# rp = relative performance hint. + +for nvme in {0..15}; do + dev="/dev/nvme${nvme}" + dev_ns="${dev}n1" + nvme delete-ns $dev -n 0x1 + nvme reset $dev + nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 + # selects LBA format index 1 (4K) and no secure erase, just format. + nvme attach-ns $dev -n 0x1 -c 0x41 + nvme format $dev_ns --lbaf=1 --ses=0 --force + nvme reset $dev + nvme id-ns $dev_ns +done + +for nvme in nvme{2..15}n1; do + dev="/dev/${nvme}" + mnt="/mnt/${nvme}" + mkfs.ext4 "${dev}" + if [ ! -d "${mnt}" ]; then + mkdir "${mnt}" + fi + sudo mount "${dev}" "${mnt}" +done + +echo "Done. All NVMe devices have been re-bound in sorted PCI address order." +for i in $(seq 0 15); do + echo -n "nvme$i: " + cat /sys/class/nvme/nvme$i/address 2>/dev/null || echo "not found" +done + +pushd /usr/share/daos/spdk/scripts/ +export PCI_ALLOWED="0000:2b:00.0 0000:2c:00.0" +sudo ./setup.sh + +daos_server nvme scan From 5b7b9321ba352396f8fb8987c4ca82f1070b2963 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 10 Dec 2025 21:16:59 +0000 Subject: [PATCH 15/40] Increase max repetition number for cluster sanity check Signed-off-by: Tomasz Gromadzki --- src/tests/ftest/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index a62abe0a508..e6cc07263e4 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -33,7 +33,7 @@ from util.yaml_utils import YamlException DEFAULT_LOGS_THRESHOLD = "2150M" # 2.1G -MAX_CI_REPETITIONS = 10 +MAX_CI_REPETITIONS = 99 class LaunchError(Exception): From a77ad8b363fbb4d70bc353967e5a0d09f0fd8425 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 11 Dec 2025 15:30:09 +0000 Subject: [PATCH 16/40] Update script to enable two devices per numa node Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 2 +- utils/tools/nvme-full-init.sh | 169 ++++++++++++++++++++++++++++++++ utils/tools/unbind_bind_nvme.sh | 112 --------------------- 3 files changed, 170 insertions(+), 113 deletions(-) create mode 100644 utils/tools/nvme-full-init.sh delete mode 100644 utils/tools/unbind_bind_nvme.sh diff --git a/Jenkinsfile b/Jenkinsfile index e4f206083bf..ec4638b306d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -390,7 +390,7 @@ pipeline { description: 'Label to use for NLT tests') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_LABEL', // defaultValue: 'ci_node-hdr-200_202-205X', - defaultValue: 'ci_node-hdr-210_212-215X', + defaultValue: 'ci_node-hdr-210_212-219X', description: 'Label to use for the Functional Hardware Medium (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL', defaultValue: 'ci_ofed5', diff --git a/utils/tools/nvme-full-init.sh b/utils/tools/nvme-full-init.sh new file mode 100644 index 00000000000..dfe1bdcc4b6 --- /dev/null +++ b/utils/tools/nvme-full-init.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# +# Copyright 2025 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# Full reinitialization of all NVMe devices on the DAOS cluster node +# - unmount all NVMe devices +# - unbind all NVMe devices and re-bind them to the nvme driver in sorted PCI address order. +# - create new logical block device (namespace) on each NVMe drive +# - mount all except two logical devices to build configuration required for DAOS cluster node +# - initialize two logical devices to work with SPDK (via vfio-pci driver) +# - verify configuration using the `daos_server nvme scan` command. +# +#set -e +#set -x +set -eu + +NVME_MAX_GLOBAL=15 +function nvme_count_devices { + local count + count=$(lspci -D | grep -E 'Non-Volatile memory controller' | wc -l) + echo $count +} + +function nvme_unmount_all { + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints (if any)..." + set +e # we can try to unmount unexisted disks + for i in $(seq 0 $NVME_MAX); do + nvme="nvme${i}n1" + dev="/dev/${nvme}" + mnt="/mnt/${nvme}" + if [ -b "$dev" ]; then + # Get mountpoint (empty if not mounted) + mp=$(lsblk -nr -o MOUNTPOINT "$dev") + if [ -n "$mp" ]; then + echo "Unmounting $dev from $mp" + sudo umount -f "$dev" 2>/dev/null || sudo umount -f "$mp" 2>/dev/null + else + echo "Unmounting ${mnt}" + sudo umount -f "${mnt}" 2>/dev/null + fi + elif [ -d "${mnt}" ]; then + echo "Force umount of ${mnt}" + sudo umount -f "${mnt}" 2>/dev/null + fi + done + set -e +} + +function nvme_bind_all_in_order { + # Find all PCI addresses for NVMe controllers + local nvme_pcie_addrs + nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') + + if [ -z "$nvme_pcie_addrs" ]; then + echo "No NVMe PCI devices found." + return 1 + fi + + #echo "Found PCI NVMe addresses:" + #echo "$nvme_pcie_addrs" + #echo + + # Unbind all NVMe devices + echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." + set +e # it's ok if a device isn't bound to one of the drivers + for addr in $nvme_pcie_addrs; do + echo "Unbinding $addr" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + done + set -e + + echo + echo "Binding NVMe devices to nvme driver in sorted order..." + for addr in $(echo "$nvme_pcie_addrs" | sort); do + echo "Binding $addr" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind > /dev/null 2>&1 + done +} + + +function nvme_recreate_namespace { + set +e +# lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks +# lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) +# lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata +# lbaf 3 : ms:8 lbads:12 rp:0x2 → 4K + 8B metadata +# lbaf 4 : ms:64 lbads:12 rp:0x3 → 4K + 64B metadata +# lbads = log2(block size). +# 9 → 2⁹ = 512 bytes +# 12 → 2¹² = 4096 bytes (4K) +# ms = metadata size per block (0, 8, or 64 bytes). +# rp = relative performance hint. + + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + for i in $(seq 0 $NVME_MAX); do + dev="/dev/nvme${i}" + dev_ns="${dev}n1" + echo "Recreating namespace on $dev ..." + nvme delete-ns $dev -n 0x1 + nvme reset $dev + nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 + nvme attach-ns $dev -n 0x1 -c 0x41 + # selects LBA format index 0 (512BK) and no secure erase, just format. + nvme format $dev_ns --lbaf=0 --ses=0 --force + nvme reset $dev + nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" + done + set -e +} + +SPDK_PCI_ALLOWED="" +function nvme_reserve_2_disk_per_numa { + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + local numa_node_0=0 + local numa_node_1=0 + for i in $(seq 0 $NVME_MAX); do + nvme="nvme${i}" + numa_path="/sys/class/nvme/${nvme}/numa_node" + dev="/dev/${nvme}n1" + mnt="/mnt/${nvme}n1" + numa_node="$(cat "$numa_path")" + pci_addr=$(basename "$(readlink -f /sys/class/nvme/${nvme}/device)") + echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" + if [ "$numa_node" -eq 0 ]; then + ((numa_node_0++)) || true + if [ "$numa_node_0" -lt 3 ]; then + SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " + echo "NUMA0: ${nvme} -> ${pci_addr}" + continue + fi + else + ((numa_node_1++)) || true + if [ "$numa_node_1" -lt 3 ]; then + SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " + echo "NUMA1: ${nvme} -> ${pci_addr}" + continue + fi + fi + sudo mkfs.ext4 -F "${dev}" + if [ ! -d "${mnt}" ]; then + sudo mkdir -p "${mnt}" + fi + sudo mount "${dev}" "${mnt}" + done + SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space +} + +nvme_count=$(nvme_count_devices) +((nvme_count--)) || true +nvme_unmount_all $nvme_count +nvme_bind_all_in_order $nvme_count +nvme_recreate_namespace $nvme_count +nvme_reserve_2_disk_per_numa $nvme_count + +#echo "Done. All NVMe devices have been re-bound in sorted PCI address order." +#for i in $(seq 0 $NVME_MAX_GLOBAL); do +# echo -n "nvme$i: " +# cat /sys/class/nvme/nvme$i/address 2>/dev/null || echo "not found" +#done + +pushd /usr/share/daos/spdk/scripts/ +export PCI_ALLOWED="$SPDK_PCI_ALLOWED" +sudo ./setup.sh + +daos_server nvme scan diff --git a/utils/tools/unbind_bind_nvme.sh b/utils/tools/unbind_bind_nvme.sh deleted file mode 100644 index 97f273a3ade..00000000000 --- a/utils/tools/unbind_bind_nvme.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# -# Copyright 2025 Hewlett Packard Enterprise Development LP -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -# -# Full reinitialization of all NVMe devices on the DAOS cluster node -# - unmount all NVMe devices -# - unbind all NVMe devices and re-bind them to the nvme driver in sorted PCI address order. -# - create new logical block device (namespace) on each NVMe drive -# - mount all except two logical devices to build configuration required for DAOS cluster node -# - initialize two logical devices to work with SPDK (via vfio-pci driver) -# - verify configuration using the `daos_server nvme scan` command. -# -# set -e -# set -x -set -u - -echo "Unmounting all /dev/nvme[0-15]n1 mountpoints (if any)..." - -for nvme in nvme{0..15}n1; do - dev="/dev/${nvme}" - mnt="/mnt/${nvme}" - if [ -b "$dev" ]; then - # Get mountpoint (empty if not mounted) - mp=$(lsblk -nr -o MOUNTPOINT "$dev") - if [ -n "$mp" ]; then - echo "Unmounting $dev from $mp" - sudo umount -f "$dev" || sudo umount -f "$mp" - else - echo "Umounting ${mnt}" - sudo umount -f "${mnt}" - fi - fi - if [ -d "${mnt}" ]; then - echo "Force umount of ${mnt}" - sudo umount -f "${mnt}" - fi -done - -# Find all PCI addresses for NVMe controllers -nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') - -if [ -z "$nvme_pcie_addrs" ]; then - echo "No NVMe PCI devices found." - exit 1 -fi - -echo "Found PCI NVMe addresses:" -echo "$nvme_pcie_addrs" -echo - -# Unbind all NVMe devices -echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." -for addr in $nvme_pcie_addrs; do - echo "Unbinding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind -done - -echo -echo "Binding NVMe devices to nvme driver in sorted order..." -for addr in $(echo "$nvme_pcie_addrs" | sort); do - echo "Binding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind -done - -# lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks -# lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) -# lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata -# lbaf 3 : ms:8 lbads:12 rp:0x2 → 4K + 8B metadata -# lbaf 4 : ms:64 lbads:12 rp:0x3 → 4K + 64B metadata -# lbads = log2(block size). -# 9 → 2⁹ = 512 bytes -# 12 → 2¹² = 4096 bytes (4K) -# ms = metadata size per block (0, 8, or 64 bytes). -# rp = relative performance hint. - -for nvme in {0..15}; do - dev="/dev/nvme${nvme}" - dev_ns="${dev}n1" - nvme delete-ns $dev -n 0x1 - nvme reset $dev - nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 - # selects LBA format index 1 (4K) and no secure erase, just format. - nvme attach-ns $dev -n 0x1 -c 0x41 - nvme format $dev_ns --lbaf=1 --ses=0 --force - nvme reset $dev - nvme id-ns $dev_ns -done - -for nvme in nvme{2..15}n1; do - dev="/dev/${nvme}" - mnt="/mnt/${nvme}" - mkfs.ext4 "${dev}" - if [ ! -d "${mnt}" ]; then - mkdir "${mnt}" - fi - sudo mount "${dev}" "${mnt}" -done - -echo "Done. All NVMe devices have been re-bound in sorted PCI address order." -for i in $(seq 0 15); do - echo -n "nvme$i: " - cat /sys/class/nvme/nvme$i/address 2>/dev/null || echo "not found" -done - -pushd /usr/share/daos/spdk/scripts/ -export PCI_ALLOWED="0000:2b:00.0 0000:2c:00.0" -sudo ./setup.sh - -daos_server nvme scan From 5d04c1871cb432c48b1f0147c1826be5d81ff296 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 11 Dec 2025 17:32:22 +0000 Subject: [PATCH 17/40] Full NVME devices configuration during node provisioning Cancel-prev-build: false Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 154 ++++++++++++++++++ utils/tools/nvme-full-init.sh | 32 ++-- 2 files changed, 171 insertions(+), 15 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 37ac6f23aaa..51e69052b09 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -69,6 +69,160 @@ if ! post_provision_config_nodes; then exit "$rc" fi +# This workaround ensures that the NVMe configuration remains consistent across +# all cluster nodes. +# This prevents situations where the binding between NVMe devices and PCIe +# addresses varies from restart to restart, resulting in error messages such as +# "Failed to initialise SSD: [xxxx:xx:xx.x]' when DAOS engines are started. + +NVME_MAX_GLOBAL=15 +function nvme_count_devices { + local count + count=$(lspci -D | grep -E 'Non-Volatile memory controller' | wc -l) + echo $count +} + +function nvme_unmount_all { + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints." + set +e # we can try to unmount unexisted disks + for i in $(seq 0 $NVME_MAX); do + nvme="nvme${i}n1" + dev="/dev/${nvme}" + mnt="/mnt/${nvme}" + if [ -b "$dev" ]; then + mp=$(lsblk -nr -o MOUNTPOINT "$dev") + if [ -n "$mp" ]; then + echo "Unmounting $dev from $mp" + sudo umount -f "$dev" 2>/dev/null || sudo umount -f "$mp" 2>/dev/null + else + echo "Unmounting ${mnt}" + sudo umount -f "${mnt}" 2>/dev/null + fi + elif [ -d "${mnt}" ]; then + echo "Force umount of ${mnt}" + sudo umount -f "${mnt}" 2>/dev/null + fi + done + set -e +} + +function nvme_bind_all_in_order { + # Find all PCI addresses for NVMe controllers + local nvme_pcie_addrs + nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') + + if [ -z "$nvme_pcie_addrs" ]; then + echo "No NVMe PCI devices found." + return 1 + fi + # Unbind all NVMe devices + echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." + set +e # it's ok if a device isn't bound to one of the drivers + for addr in $nvme_pcie_addrs; do + echo "Unbinding $addr" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + done + set -e + + echo + # Bind all NVMe devices in order + echo "Binding NVMe devices to nvme driver in sorted order..." + for addr in $(echo "$nvme_pcie_addrs" | sort); do + echo "Binding $addr" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind > /dev/null 2>&1 + done +} + + +function nvme_recreate_namespace { + set +e +# lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks +# lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) +# lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata +# lbaf 3 : ms:8 lbads:12 rp:0x2 → 4K + 8B metadata +# lbaf 4 : ms:64 lbads:12 rp:0x3 → 4K + 64B metadata +# lbads = log2(block size). +# 9 → 2⁹ = 512 bytes +# 12 → 2¹² = 4096 bytes (4K) +# ms = metadata size per block (0, 8, or 64 bytes). +# rp = relative performance hint. + + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + for i in $(seq 0 $NVME_MAX); do + dev="/dev/nvme${i}" + dev_ns="${dev}n1" + echo "Recreating namespace on $dev ..." + nvme delete-ns $dev -n 0x1 + nvme reset $dev + nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 + nvme attach-ns $dev -n 0x1 -c 0x41 + # selects LBA format index 0 (512BK) and no secure erase, just format. + nvme format $dev_ns --lbaf=0 --ses=0 --force + nvme reset $dev + nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" + done + set -e +} + +SPDK_PCI_ALLOWED="" +function nvme_reserve_2_disk_per_numa { + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + local numa_node_0=0 + local numa_node_1=0 + for i in $(seq 0 $NVME_MAX); do + nvme="nvme${i}" + numa_path="/sys/class/nvme/${nvme}/numa_node" + dev="/dev/${nvme}n1" + mnt="/mnt/${nvme}n1" + numa_node="$(cat "$numa_path")" + pci_addr=$(basename "$(readlink -f /sys/class/nvme/${nvme}/device)") + echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" + if [ "$numa_node" -eq 0 ]; then + ((numa_node_0++)) || true + if [ "$numa_node_0" -lt 3 ]; then + SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " + echo "NUMA0: ${nvme} -> ${pci_addr}" + continue + fi + else + ((numa_node_1++)) || true + if [ "$numa_node_1" -lt 3 ]; then + SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " + echo "NUMA1: ${nvme} -> ${pci_addr}" + continue + fi + fi + sudo mkfs.ext4 -F "${dev}" + if [ ! -d "${mnt}" ]; then + sudo mkdir -p "${mnt}" + fi + sudo mount "${dev}" "${mnt}" + done + SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space +} + +nvme_count=$(nvme_count_devices) +if [ "$nvme_count" -gt 1 ]; then + ((nvme_count--)) || true + nvme_unmount_all $nvme_count + nvme_bind_all_in_order + nvme_recreate_namespace $nvme_count + nvme_reserve_2_disk_per_numa $nvme_count + + if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then + export PCI_ALLOWED="$SPDK_PCI_ALLOWED" + pushd /usr/share/daos/spdk/scripts/ + set +e + sudo ./setup.sh + set -e + popd + else + echo "Required spdk/scripts/setup.sh not found!" + fi +fi + # Workaround to enable binding devices back to nvme or vfio-pci after they are unbound from vfio-pci # to nvme. Sometimes the device gets unbound from vfio-pci, but it is not removed the iommu group # for that device and future bindings to the device do not work, resulting in messages like, "NVMe diff --git a/utils/tools/nvme-full-init.sh b/utils/tools/nvme-full-init.sh index dfe1bdcc4b6..3b86506cc14 100644 --- a/utils/tools/nvme-full-init.sh +++ b/utils/tools/nvme-full-init.sh @@ -56,7 +56,7 @@ function nvme_bind_all_in_order { if [ -z "$nvme_pcie_addrs" ]; then echo "No NVMe PCI devices found." - return 1 + return 0 fi #echo "Found PCI NVMe addresses:" @@ -150,20 +150,22 @@ function nvme_reserve_2_disk_per_numa { } nvme_count=$(nvme_count_devices) -((nvme_count--)) || true -nvme_unmount_all $nvme_count -nvme_bind_all_in_order $nvme_count -nvme_recreate_namespace $nvme_count -nvme_reserve_2_disk_per_numa $nvme_count +if [ "$nvme_count" -gt 1 ]; then + ((nvme_count--)) || true + nvme_unmount_all $nvme_count + nvme_bind_all_in_order $nvme_count + nvme_recreate_namespace $nvme_count + nvme_reserve_2_disk_per_numa $nvme_count -#echo "Done. All NVMe devices have been re-bound in sorted PCI address order." -#for i in $(seq 0 $NVME_MAX_GLOBAL); do -# echo -n "nvme$i: " -# cat /sys/class/nvme/nvme$i/address 2>/dev/null || echo "not found" -#done + #echo "Done. All NVMe devices have been re-bound in sorted PCI address order." + #for i in $(seq 0 $NVME_MAX_GLOBAL); do + # echo -n "nvme$i: " + # cat /sys/class/nvme/nvme$i/address 2>/dev/null || echo "not found" + #done -pushd /usr/share/daos/spdk/scripts/ -export PCI_ALLOWED="$SPDK_PCI_ALLOWED" -sudo ./setup.sh + pushd /usr/share/daos/spdk/scripts/ + export PCI_ALLOWED="$SPDK_PCI_ALLOWED" + sudo ./setup.sh -daos_server nvme scan + daos_server nvme scan +fi \ No newline at end of file From 0ad5d6eb9a960f9371ebda52f4014116ac46637c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 12 Dec 2025 21:00:04 +0000 Subject: [PATCH 18/40] Speed up test by les repetitions Signed-off-by: Tomasz Gromadzki --- ci/provisioning/post_provision_config_nodes.sh | 8 ++++++-- src/tests/ftest/pool/create_all_hw.yaml | 2 +- utils/tools/nvme-full-init.sh | 16 ++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 51e69052b09..98e9f417d7b 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -121,8 +121,12 @@ function nvme_bind_all_in_order { set +e # it's ok if a device isn't bound to one of the drivers for addr in $nvme_pcie_addrs; do echo "Unbinding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + if [ -f "/sys/bus/pci/drivers/nvme/${addr}" ]; then + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 + fi + if [ -f "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + fi done set -e diff --git a/src/tests/ftest/pool/create_all_hw.yaml b/src/tests/ftest/pool/create_all_hw.yaml index c4fcf5cb770..254dd930470 100644 --- a/src/tests/ftest/pool/create_all_hw.yaml +++ b/src/tests/ftest/pool/create_all_hw.yaml @@ -16,7 +16,7 @@ test_one_pool_hw: nvme: 1048576 # 1MiB test_recycle_pools_hw: - pool_count: 50 + pool_count: 1 deltas: scm: 18874368 # 18MiB nvme: 0 # 0MiB diff --git a/utils/tools/nvme-full-init.sh b/utils/tools/nvme-full-init.sh index 3b86506cc14..d523821c685 100644 --- a/utils/tools/nvme-full-init.sh +++ b/utils/tools/nvme-full-init.sh @@ -63,15 +63,23 @@ function nvme_bind_all_in_order { #echo "$nvme_pcie_addrs" #echo + for dir in /sys/class/nvme/*/; do + echo "$dir: $(ls -la ${dir} | grep device | awk -F'-> ' '{print $2}' | sed 's|.*/||')" + done + # Unbind all NVMe devices echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." - set +e # it's ok if a device isn't bound to one of the drivers + #set +e # it's ok if a device isn't bound to one of the drivers for addr in $nvme_pcie_addrs; do echo "Unbinding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + if [ -f "/sys/bus/pci/drivers/nvme/${addr}" ]; then + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 + fi + if [ -f "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + fi done - set -e + #set -e echo echo "Binding NVMe devices to nvme driver in sorted order..." From 504ad56086bda3b6b96596b9c4116f5769230b88 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 12 Dec 2025 21:01:55 +0000 Subject: [PATCH 19/40] Avoid error in case of unbinding failure Signed-off-by: Tomasz Gromadzki --- utils/tools/nvme-full-init.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/tools/nvme-full-init.sh b/utils/tools/nvme-full-init.sh index d523821c685..a192bc3dcaf 100644 --- a/utils/tools/nvme-full-init.sh +++ b/utils/tools/nvme-full-init.sh @@ -66,10 +66,10 @@ function nvme_bind_all_in_order { for dir in /sys/class/nvme/*/; do echo "$dir: $(ls -la ${dir} | grep device | awk -F'-> ' '{print $2}' | sed 's|.*/||')" done - + # Unbind all NVMe devices echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." - #set +e # it's ok if a device isn't bound to one of the drivers + set +e # it's ok if a device isn't bound to one of the drivers for addr in $nvme_pcie_addrs; do echo "Unbinding $addr" if [ -f "/sys/bus/pci/drivers/nvme/${addr}" ]; then @@ -79,7 +79,7 @@ function nvme_bind_all_in_order { echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 fi done - #set -e + set -e echo echo "Binding NVMe devices to nvme driver in sorted order..." From f95e526219b0920f611ac84ab11b0500f4b5e9b1 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 12 Dec 2025 21:04:22 +0000 Subject: [PATCH 20/40] Run Tests without fix to collect more data Let's try to run fix with original configuration to confirm that problem still exists. Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 158 ------------------ 1 file changed, 158 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 98e9f417d7b..37ac6f23aaa 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -69,164 +69,6 @@ if ! post_provision_config_nodes; then exit "$rc" fi -# This workaround ensures that the NVMe configuration remains consistent across -# all cluster nodes. -# This prevents situations where the binding between NVMe devices and PCIe -# addresses varies from restart to restart, resulting in error messages such as -# "Failed to initialise SSD: [xxxx:xx:xx.x]' when DAOS engines are started. - -NVME_MAX_GLOBAL=15 -function nvme_count_devices { - local count - count=$(lspci -D | grep -E 'Non-Volatile memory controller' | wc -l) - echo $count -} - -function nvme_unmount_all { - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints." - set +e # we can try to unmount unexisted disks - for i in $(seq 0 $NVME_MAX); do - nvme="nvme${i}n1" - dev="/dev/${nvme}" - mnt="/mnt/${nvme}" - if [ -b "$dev" ]; then - mp=$(lsblk -nr -o MOUNTPOINT "$dev") - if [ -n "$mp" ]; then - echo "Unmounting $dev from $mp" - sudo umount -f "$dev" 2>/dev/null || sudo umount -f "$mp" 2>/dev/null - else - echo "Unmounting ${mnt}" - sudo umount -f "${mnt}" 2>/dev/null - fi - elif [ -d "${mnt}" ]; then - echo "Force umount of ${mnt}" - sudo umount -f "${mnt}" 2>/dev/null - fi - done - set -e -} - -function nvme_bind_all_in_order { - # Find all PCI addresses for NVMe controllers - local nvme_pcie_addrs - nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') - - if [ -z "$nvme_pcie_addrs" ]; then - echo "No NVMe PCI devices found." - return 1 - fi - # Unbind all NVMe devices - echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." - set +e # it's ok if a device isn't bound to one of the drivers - for addr in $nvme_pcie_addrs; do - echo "Unbinding $addr" - if [ -f "/sys/bus/pci/drivers/nvme/${addr}" ]; then - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 - fi - if [ -f "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 - fi - done - set -e - - echo - # Bind all NVMe devices in order - echo "Binding NVMe devices to nvme driver in sorted order..." - for addr in $(echo "$nvme_pcie_addrs" | sort); do - echo "Binding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind > /dev/null 2>&1 - done -} - - -function nvme_recreate_namespace { - set +e -# lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks -# lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) -# lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata -# lbaf 3 : ms:8 lbads:12 rp:0x2 → 4K + 8B metadata -# lbaf 4 : ms:64 lbads:12 rp:0x3 → 4K + 64B metadata -# lbads = log2(block size). -# 9 → 2⁹ = 512 bytes -# 12 → 2¹² = 4096 bytes (4K) -# ms = metadata size per block (0, 8, or 64 bytes). -# rp = relative performance hint. - - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - for i in $(seq 0 $NVME_MAX); do - dev="/dev/nvme${i}" - dev_ns="${dev}n1" - echo "Recreating namespace on $dev ..." - nvme delete-ns $dev -n 0x1 - nvme reset $dev - nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 - nvme attach-ns $dev -n 0x1 -c 0x41 - # selects LBA format index 0 (512BK) and no secure erase, just format. - nvme format $dev_ns --lbaf=0 --ses=0 --force - nvme reset $dev - nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" - done - set -e -} - -SPDK_PCI_ALLOWED="" -function nvme_reserve_2_disk_per_numa { - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - local numa_node_0=0 - local numa_node_1=0 - for i in $(seq 0 $NVME_MAX); do - nvme="nvme${i}" - numa_path="/sys/class/nvme/${nvme}/numa_node" - dev="/dev/${nvme}n1" - mnt="/mnt/${nvme}n1" - numa_node="$(cat "$numa_path")" - pci_addr=$(basename "$(readlink -f /sys/class/nvme/${nvme}/device)") - echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" - if [ "$numa_node" -eq 0 ]; then - ((numa_node_0++)) || true - if [ "$numa_node_0" -lt 3 ]; then - SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " - echo "NUMA0: ${nvme} -> ${pci_addr}" - continue - fi - else - ((numa_node_1++)) || true - if [ "$numa_node_1" -lt 3 ]; then - SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " - echo "NUMA1: ${nvme} -> ${pci_addr}" - continue - fi - fi - sudo mkfs.ext4 -F "${dev}" - if [ ! -d "${mnt}" ]; then - sudo mkdir -p "${mnt}" - fi - sudo mount "${dev}" "${mnt}" - done - SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space -} - -nvme_count=$(nvme_count_devices) -if [ "$nvme_count" -gt 1 ]; then - ((nvme_count--)) || true - nvme_unmount_all $nvme_count - nvme_bind_all_in_order - nvme_recreate_namespace $nvme_count - nvme_reserve_2_disk_per_numa $nvme_count - - if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then - export PCI_ALLOWED="$SPDK_PCI_ALLOWED" - pushd /usr/share/daos/spdk/scripts/ - set +e - sudo ./setup.sh - set -e - popd - else - echo "Required spdk/scripts/setup.sh not found!" - fi -fi - # Workaround to enable binding devices back to nvme or vfio-pci after they are unbound from vfio-pci # to nvme. Sometimes the device gets unbound from vfio-pci, but it is not removed the iommu group # for that device and future bindings to the device do not work, resulting in messages like, "NVMe From b915253d5730a7b71b785e7c002f2d9444f13cc2 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 15 Dec 2025 13:02:26 +0000 Subject: [PATCH 21/40] Run tests with wiped namespaces Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Revert "Run Tests without fix to collect more data" This reverts commit 2bae9ce695614605e779cf98390727ef2229ec60. Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 37ac6f23aaa..287e974584b 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -69,6 +69,164 @@ if ! post_provision_config_nodes; then exit "$rc" fi +# This workaround ensures that the NVMe configuration remains consistent across +# all cluster nodes. +# This prevents situations where the binding between NVMe devices and PCIe +# addresses varies from restart to restart, resulting in error messages such as +# "Failed to initialise SSD: [xxxx:xx:xx.x]' when DAOS engines are started. + +NVME_MAX_GLOBAL=15 +function nvme_count_devices { + local count + count=$(lspci -D | grep -E 'Non-Volatile memory controller' | wc -l) + echo $count +} + +function nvme_unmount_all { + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints." + set +e # we can try to unmount unexisted disks + for i in $(seq 0 $NVME_MAX); do + nvme="nvme${i}n1" + dev="/dev/${nvme}" + mnt="/mnt/${nvme}" + if [ -b "$dev" ]; then + mp=$(lsblk -nr -o MOUNTPOINT "$dev") + if [ -n "$mp" ]; then + echo "Unmounting $dev from $mp" + sudo umount -f "$dev" 2>/dev/null || sudo umount -f "$mp" 2>/dev/null + else + echo "Unmounting ${mnt}" + sudo umount -f "${mnt}" 2>/dev/null + fi + elif [ -d "${mnt}" ]; then + echo "Force umount of ${mnt}" + sudo umount -f "${mnt}" 2>/dev/null + fi + done + set -e +} + +function nvme_bind_all_in_order { + # Find all PCI addresses for NVMe controllers + local nvme_pcie_addrs + nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') + + if [ -z "$nvme_pcie_addrs" ]; then + echo "No NVMe PCI devices found." + return 1 + fi + # Unbind all NVMe devices + echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." + set +e # it's ok if a device isn't bound to one of the drivers + for addr in $nvme_pcie_addrs; do + echo "Unbinding $addr" + if [ -f "/sys/bus/pci/drivers/nvme/${addr}" ]; then + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 + fi + if [ -f "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + fi + done + set -e + + echo + # Bind all NVMe devices in order + echo "Binding NVMe devices to nvme driver in sorted order..." + for addr in $(echo "$nvme_pcie_addrs" | sort); do + echo "Binding $addr" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind > /dev/null 2>&1 + done +} + + +function nvme_recreate_namespace { + set +e +# lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks +# lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) +# lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata +# lbaf 3 : ms:8 lbads:12 rp:0x2 → 4K + 8B metadata +# lbaf 4 : ms:64 lbads:12 rp:0x3 → 4K + 64B metadata +# lbads = log2(block size). +# 9 → 2⁹ = 512 bytes +# 12 → 2¹² = 4096 bytes (4K) +# ms = metadata size per block (0, 8, or 64 bytes). +# rp = relative performance hint. + + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + for i in $(seq 0 $NVME_MAX); do + dev="/dev/nvme${i}" + dev_ns="${dev}n1" + echo "Recreating namespace on $dev ..." + nvme delete-ns $dev -n 0x1 + nvme reset $dev + nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 + nvme attach-ns $dev -n 0x1 -c 0x41 + # selects LBA format index 0 (512BK) and no secure erase, just format. + nvme format $dev_ns --lbaf=0 --ses=0 --force + nvme reset $dev + nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" + done + set -e +} + +SPDK_PCI_ALLOWED="" +function nvme_reserve_2_disk_per_numa { + local NVME_MAX=${1:-$NVME_MAX_GLOBAL} + local numa_node_0=0 + local numa_node_1=0 + for i in $(seq 0 $NVME_MAX); do + nvme="nvme${i}" + numa_path="/sys/class/nvme/${nvme}/numa_node" + dev="/dev/${nvme}n1" + mnt="/mnt/${nvme}n1" + numa_node="$(cat "$numa_path")" + pci_addr=$(basename "$(readlink -f /sys/class/nvme/${nvme}/device)") + echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" + if [ "$numa_node" -eq 0 ]; then + ((numa_node_0++)) || true + if [ "$numa_node_0" -lt 3 ]; then + SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " + echo "NUMA0: ${nvme} -> ${pci_addr}" + continue + fi + else + ((numa_node_1++)) || true + if [ "$numa_node_1" -lt 3 ]; then + SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " + echo "NUMA1: ${nvme} -> ${pci_addr}" + continue + fi + fi + sudo mkfs.ext4 -F "${dev}" + if [ ! -d "${mnt}" ]; then + sudo mkdir -p "${mnt}" + fi + sudo mount "${dev}" "${mnt}" + done + SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space +} + +nvme_count=$(nvme_count_devices) +if [ "$nvme_count" -gt 1 ]; then + ((nvme_count--)) || true +# nvme_unmount_all $nvme_count +# nvme_bind_all_in_order + nvme_recreate_namespace $nvme_count +# nvme_reserve_2_disk_per_numa $nvme_count + +# if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then +# export PCI_ALLOWED="$SPDK_PCI_ALLOWED" +# pushd /usr/share/daos/spdk/scripts/ +# set +e +# sudo ./setup.sh +# set -e +# popd +# else +# echo "Required spdk/scripts/setup.sh not found!" +# fi +fi + # Workaround to enable binding devices back to nvme or vfio-pci after they are unbound from vfio-pci # to nvme. Sometimes the device gets unbound from vfio-pci, but it is not removed the iommu group # for that device and future bindings to the device do not work, resulting in messages like, "NVMe From 1b0f7977cc10ae8b10fe5a3211dbbe8b2517a23d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 16 Dec 2025 19:58:25 +0000 Subject: [PATCH 22/40] Fix: clamav dependency mismatch clamav must be removed as required dependency is not avaialble: "nothing provides libcrypto.so.3(OPENSSL_3.4.0)(64bit)" Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 6 +++--- ci/provisioning/post_provision_config_nodes_EL.sh | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ec4638b306d..be4102c785b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -389,8 +389,8 @@ pipeline { defaultValue: 'ci_nlt_1', description: 'Label to use for NLT tests') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_LABEL', - // defaultValue: 'ci_node-hdr-200_202-205X', - defaultValue: 'ci_node-hdr-210_212-219X', + defaultValue: 'ci_node-hdr-200_202-205X', + // defaultValue: 'ci_node-hdr-210_212-219X', description: 'Label to use for the Functional Hardware Medium (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL', defaultValue: 'ci_ofed5', @@ -405,7 +405,7 @@ pipeline { defaultValue: 'ci_nvme9', description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION', - defaultValue: 'el8.8', + defaultValue: 'el9.5', description: 'Label to use for 5 node Functional Hardware Medium (MD on SSD) stages') string(name: 'CI_STORAGE_PREP_LABEL', defaultValue: '', diff --git a/ci/provisioning/post_provision_config_nodes_EL.sh b/ci/provisioning/post_provision_config_nodes_EL.sh index 75e1d7934e3..505d445fca3 100644 --- a/ci/provisioning/post_provision_config_nodes_EL.sh +++ b/ci/provisioning/post_provision_config_nodes_EL.sh @@ -18,6 +18,9 @@ set +e set -e # Seems to be needed to fix some issues. dnf -y reinstall sssd-common + # Seems to be required until https://daosio.atlassian.net/browse/DAOS-18358 + # is fixed. + dnf -y remove clamav-lib } group_repo_post() { From 36cb784eef8068e8ac237bb6f3d582ff89d159b2 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 16 Dec 2025 21:24:34 +0000 Subject: [PATCH 23/40] Fix: for el9.5 strict order of NVMe devices is needed Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 287e974584b..e46f0c394bd 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -170,6 +170,9 @@ function nvme_recreate_namespace { set -e } +# FOR now limit to 2 devices per CPU NUMA node +: "${DAOS_CI_NVME_NUMA_LIMIT:=2}" + SPDK_PCI_ALLOWED="" function nvme_reserve_2_disk_per_numa { local NVME_MAX=${1:-$NVME_MAX_GLOBAL} @@ -185,14 +188,14 @@ function nvme_reserve_2_disk_per_numa { echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" if [ "$numa_node" -eq 0 ]; then ((numa_node_0++)) || true - if [ "$numa_node_0" -lt 3 ]; then + if [ "$numa_node_0" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " echo "NUMA0: ${nvme} -> ${pci_addr}" continue fi else ((numa_node_1++)) || true - if [ "$numa_node_1" -lt 3 ]; then + if [ "$numa_node_1" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " echo "NUMA1: ${nvme} -> ${pci_addr}" continue @@ -211,20 +214,20 @@ nvme_count=$(nvme_count_devices) if [ "$nvme_count" -gt 1 ]; then ((nvme_count--)) || true # nvme_unmount_all $nvme_count -# nvme_bind_all_in_order + nvme_bind_all_in_order nvme_recreate_namespace $nvme_count -# nvme_reserve_2_disk_per_numa $nvme_count + nvme_reserve_2_disk_per_numa $nvme_count -# if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then -# export PCI_ALLOWED="$SPDK_PCI_ALLOWED" -# pushd /usr/share/daos/spdk/scripts/ -# set +e -# sudo ./setup.sh -# set -e -# popd -# else -# echo "Required spdk/scripts/setup.sh not found!" -# fi + if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then + export PCI_ALLOWED="$SPDK_PCI_ALLOWED" + pushd /usr/share/daos/spdk/scripts/ + set +e + sudo ./setup.sh + set -e + popd + else + echo "Required spdk/scripts/setup.sh not found!" + fi fi # Workaround to enable binding devices back to nvme or vfio-pci after they are unbound from vfio-pci @@ -236,9 +239,6 @@ if lspci | grep -i nvme; then daos_server nvme reset && rmmod vfio_pci && modprobe vfio_pci fi -# FOR now limit to 2 devices per CPU NUMA node -: "${DAOS_CI_NVME_NUMA_LIMIT:=2}" - function mount_nvme_drive { local drive="$1" file_system=$(file -sL "/dev/$drive") From 168cd133c49926dde16056b4790f2e6ae1d96678 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 16 Dec 2025 21:30:20 +0000 Subject: [PATCH 24/40] Jenkins debugging Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index be4102c785b..79a6cd682b8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -190,6 +190,9 @@ Boolean skip_pragma_set(String name, String def_val='false') { Boolean skip_build_stage(String distro='', String compiler='gcc') { // Skip the stage if the CI__NOBUILD parameter is set + println("[${env.STAGE_NAME}] --- skip_build_stage diagnostics ---") + println(" Input: distro='$distro', compiler='$compiler'") + if (distro) { if (startedByUser() && paramsValue("CI_${distro}_NOBUILD", false)) { println("[${env.STAGE_NAME}] Skipping build stage due to CI_${distro}_NOBUILD") @@ -202,11 +205,18 @@ Boolean skip_build_stage(String distro='', String compiler='gcc') { if (distro && compiler) { pragma_names << "build-${distro}-${compiler}" } - Boolean any_pragma_skip = pragma_names.any { name -> skip_pragma_set(name) } + + println(" Pragma names to check: ${pragma_names}") + + // Check if any of the skip pragmas are set + def pragma_checks = pragma_names.collectEntries { name -> [(name): skip_pragma_set(name)] } + println(" Pragma flags: ${pragma_checks}") + + boolean any_pragma_skip = pragma_checks.values().any { it } + if (any_pragma_skip) { - println("[${env.STAGE_NAME}] Skipping build stage for due to Skip-[${pragma_names}] pragma") + println("[${env.STAGE_NAME}] Skipping build stage due to Skip-[${pragma_names.join(', ')}] pragma(s), matched: ${pragma_checks.findAll { k,v -> v }.keySet()}") return true - } // Skip the stage if a specific DAOS RPM version is specified if (rpmTestVersion() != '') { @@ -215,6 +225,8 @@ Boolean skip_build_stage(String distro='', String compiler='gcc') { } // Otherwise run the build stage + // Default: don't skip + println("[${env.STAGE_NAME}] Skipping NOT triggered. Build stage will run.") return false } From 554b4b21f995b359974d0373ed5ca717886c4662 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 16 Dec 2025 23:07:24 +0100 Subject: [PATCH 25/40] Revert "Jenkins debugging" This reverts commit e2ba2463f580c809a4055d44bb1b8cf3eff9d7d4. Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 18 +++--------------- ci/provisioning/post_provision_config_nodes.sh | 2 +- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 79a6cd682b8..be4102c785b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -190,9 +190,6 @@ Boolean skip_pragma_set(String name, String def_val='false') { Boolean skip_build_stage(String distro='', String compiler='gcc') { // Skip the stage if the CI__NOBUILD parameter is set - println("[${env.STAGE_NAME}] --- skip_build_stage diagnostics ---") - println(" Input: distro='$distro', compiler='$compiler'") - if (distro) { if (startedByUser() && paramsValue("CI_${distro}_NOBUILD", false)) { println("[${env.STAGE_NAME}] Skipping build stage due to CI_${distro}_NOBUILD") @@ -205,18 +202,11 @@ Boolean skip_build_stage(String distro='', String compiler='gcc') { if (distro && compiler) { pragma_names << "build-${distro}-${compiler}" } - - println(" Pragma names to check: ${pragma_names}") - - // Check if any of the skip pragmas are set - def pragma_checks = pragma_names.collectEntries { name -> [(name): skip_pragma_set(name)] } - println(" Pragma flags: ${pragma_checks}") - - boolean any_pragma_skip = pragma_checks.values().any { it } - + Boolean any_pragma_skip = pragma_names.any { name -> skip_pragma_set(name) } if (any_pragma_skip) { - println("[${env.STAGE_NAME}] Skipping build stage due to Skip-[${pragma_names.join(', ')}] pragma(s), matched: ${pragma_checks.findAll { k,v -> v }.keySet()}") + println("[${env.STAGE_NAME}] Skipping build stage for due to Skip-[${pragma_names}] pragma") return true + } // Skip the stage if a specific DAOS RPM version is specified if (rpmTestVersion() != '') { @@ -225,8 +215,6 @@ Boolean skip_build_stage(String distro='', String compiler='gcc') { } // Otherwise run the build stage - // Default: don't skip - println("[${env.STAGE_NAME}] Skipping NOT triggered. Build stage will run.") return false } diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index e46f0c394bd..f23d3b813e6 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -73,7 +73,7 @@ fi # all cluster nodes. # This prevents situations where the binding between NVMe devices and PCIe # addresses varies from restart to restart, resulting in error messages such as -# "Failed to initialise SSD: [xxxx:xx:xx.x]' when DAOS engines are started. +# "Failed to initialize SSD: [xxxx:xx:xx.x]' when DAOS engines are started. NVME_MAX_GLOBAL=15 function nvme_count_devices { From 656ce9875f01b9d2ca6de618396270ccba1e03a0 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 Dec 2025 08:35:08 +0000 Subject: [PATCH 26/40] fix to debug issue Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- ci/provisioning/post_provision_config_nodes.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index f23d3b813e6..ed04a026e70 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -133,10 +133,12 @@ function nvme_bind_all_in_order { echo # Bind all NVMe devices in order echo "Binding NVMe devices to nvme driver in sorted order..." + set +e # for debug purpose for addr in $(echo "$nvme_pcie_addrs" | sort); do echo "Binding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind > /dev/null 2>&1 + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind done + set -e } @@ -213,7 +215,7 @@ function nvme_reserve_2_disk_per_numa { nvme_count=$(nvme_count_devices) if [ "$nvme_count" -gt 1 ]; then ((nvme_count--)) || true -# nvme_unmount_all $nvme_count + nvme_unmount_all $nvme_count nvme_bind_all_in_order nvme_recreate_namespace $nvme_count nvme_reserve_2_disk_per_numa $nvme_count From 91ef4136771b2cd525aaadb27e1635b72720f325 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 Dec 2025 10:24:56 +0000 Subject: [PATCH 27/40] fix invalid unmount Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- ci/provisioning/post_provision_config_nodes.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index ed04a026e70..7e9ae8eec3b 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -94,14 +94,14 @@ function nvme_unmount_all { mp=$(lsblk -nr -o MOUNTPOINT "$dev") if [ -n "$mp" ]; then echo "Unmounting $dev from $mp" - sudo umount -f "$dev" 2>/dev/null || sudo umount -f "$mp" 2>/dev/null - else + sudo umount -f "$dev"|| sudo umount -f "$mp" + elif [ -d "${mnt}" ]; then echo "Unmounting ${mnt}" - sudo umount -f "${mnt}" 2>/dev/null + sudo umount -f "${mnt}" fi elif [ -d "${mnt}" ]; then echo "Force umount of ${mnt}" - sudo umount -f "${mnt}" 2>/dev/null + sudo umount -f "${mnt}" fi done set -e From 607065e0c0979fd2d820e26bb1fc03505bb76760 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 Dec 2025 13:17:57 +0000 Subject: [PATCH 28/40] Fix - -f does not detect symlink to directories Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 32 ++++++--- utils/tools/nvme-full-init.sh | 71 +++++++++++-------- 2 files changed, 65 insertions(+), 38 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 7e9ae8eec3b..7cdf9fc3f1e 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -103,6 +103,7 @@ function nvme_unmount_all { echo "Force umount of ${mnt}" sudo umount -f "${mnt}" fi + rm -rf $mnt done set -e } @@ -116,16 +117,27 @@ function nvme_bind_all_in_order { echo "No NVMe PCI devices found." return 1 fi + + #echo "Found PCI NVMe addresses:" + #echo "$nvme_pcie_addrs" + #echo + + for dir in /sys/class/nvme/*/; do + numa=$(cat ${dir}numa_node) + echo "$numa $dir: $(ls -la ${dir} | grep device | awk -F'-> ' '{print $2}' | sed 's|.*/||')" + done + # Unbind all NVMe devices echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." set +e # it's ok if a device isn't bound to one of the drivers for addr in $nvme_pcie_addrs; do - echo "Unbinding $addr" - if [ -f "/sys/bus/pci/drivers/nvme/${addr}" ]; then - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 + if [ -e "/sys/bus/pci/drivers/nvme/${addr}" ]; then + echo "Unbinding $addr from nvme" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind fi - if [ -f "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + if [ -e "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then + echo "Unbinding $addr from vfio-pci" + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind fi done set -e @@ -204,10 +216,10 @@ function nvme_reserve_2_disk_per_numa { fi fi sudo mkfs.ext4 -F "${dev}" - if [ ! -d "${mnt}" ]; then - sudo mkdir -p "${mnt}" - fi - sudo mount "${dev}" "${mnt}" +# if [ ! -d "${mnt}" ]; then +# sudo mkdir -p "${mnt}" +# fi +# sudo mount "${dev}" "${mnt}" done SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space } @@ -310,7 +322,7 @@ function nvme_limit { # Force only the desired number of NVMe devices to be seen by DAOS tests # by mounting the extra ones. -nvme_limit +# nvme_limit systemctl enable nfs-server.service systemctl start nfs-server.service diff --git a/utils/tools/nvme-full-init.sh b/utils/tools/nvme-full-init.sh index a192bc3dcaf..299cb7f31d7 100644 --- a/utils/tools/nvme-full-init.sh +++ b/utils/tools/nvme-full-init.sh @@ -1,8 +1,8 @@ #!/bin/bash # -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025 Hewlett Packard Enterprise Development LP # -# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-License-Identifier: BSD-2-Clause-Patent # # Full reinitialization of all NVMe devices on the DAOS cluster node # - unmount all NVMe devices @@ -25,26 +25,26 @@ function nvme_count_devices { function nvme_unmount_all { local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints (if any)..." + echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints." set +e # we can try to unmount unexisted disks for i in $(seq 0 $NVME_MAX); do nvme="nvme${i}n1" dev="/dev/${nvme}" mnt="/mnt/${nvme}" if [ -b "$dev" ]; then - # Get mountpoint (empty if not mounted) mp=$(lsblk -nr -o MOUNTPOINT "$dev") if [ -n "$mp" ]; then echo "Unmounting $dev from $mp" - sudo umount -f "$dev" 2>/dev/null || sudo umount -f "$mp" 2>/dev/null - else + sudo umount -f "$dev"|| sudo umount -f "$mp" + elif [ -d "${mnt}" ]; then echo "Unmounting ${mnt}" - sudo umount -f "${mnt}" 2>/dev/null + sudo umount -f "${mnt}" fi elif [ -d "${mnt}" ]; then echo "Force umount of ${mnt}" - sudo umount -f "${mnt}" 2>/dev/null + sudo umount -f "${mnt}" fi + rm -rf $mnt done set -e } @@ -56,7 +56,7 @@ function nvme_bind_all_in_order { if [ -z "$nvme_pcie_addrs" ]; then echo "No NVMe PCI devices found." - return 0 + return 1 fi #echo "Found PCI NVMe addresses:" @@ -64,29 +64,34 @@ function nvme_bind_all_in_order { #echo for dir in /sys/class/nvme/*/; do - echo "$dir: $(ls -la ${dir} | grep device | awk -F'-> ' '{print $2}' | sed 's|.*/||')" + numa=$(cat ${dir}numa_node) + echo "$numa $dir: $(ls -la ${dir} | grep device | awk -F'-> ' '{print $2}' | sed 's|.*/||')" done # Unbind all NVMe devices echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." set +e # it's ok if a device isn't bound to one of the drivers for addr in $nvme_pcie_addrs; do - echo "Unbinding $addr" - if [ -f "/sys/bus/pci/drivers/nvme/${addr}" ]; then - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind > /dev/null 2>&1 + if [ -e "/sys/bus/pci/drivers/nvme/${addr}" ]; then + echo "Unbinding $addr from nvme" + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind fi - if [ -f "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null 2>&1 + if [ -e "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then + echo "Unbinding $addr from vfio-pci" + echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind fi done set -e echo + # Bind all NVMe devices in order echo "Binding NVMe devices to nvme driver in sorted order..." + set +e # for debug purpose for addr in $(echo "$nvme_pcie_addrs" | sort); do echo "Binding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind > /dev/null 2>&1 + echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind done + set -e } @@ -115,11 +120,14 @@ function nvme_recreate_namespace { # selects LBA format index 0 (512BK) and no secure erase, just format. nvme format $dev_ns --lbaf=0 --ses=0 --force nvme reset $dev - nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" +# nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" done set -e } +# FOR now limit to 2 devices per CPU NUMA node +: "${DAOS_CI_NVME_NUMA_LIMIT:=2}" + SPDK_PCI_ALLOWED="" function nvme_reserve_2_disk_per_numa { local NVME_MAX=${1:-$NVME_MAX_GLOBAL} @@ -135,24 +143,24 @@ function nvme_reserve_2_disk_per_numa { echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" if [ "$numa_node" -eq 0 ]; then ((numa_node_0++)) || true - if [ "$numa_node_0" -lt 3 ]; then + if [ "$numa_node_0" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " echo "NUMA0: ${nvme} -> ${pci_addr}" continue fi else ((numa_node_1++)) || true - if [ "$numa_node_1" -lt 3 ]; then + if [ "$numa_node_1" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " echo "NUMA1: ${nvme} -> ${pci_addr}" continue fi fi sudo mkfs.ext4 -F "${dev}" - if [ ! -d "${mnt}" ]; then - sudo mkdir -p "${mnt}" - fi - sudo mount "${dev}" "${mnt}" +# if [ ! -d "${mnt}" ]; then +# sudo mkdir -p "${mnt}" +# fi +# sudo mount "${dev}" "${mnt}" done SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space } @@ -161,7 +169,7 @@ nvme_count=$(nvme_count_devices) if [ "$nvme_count" -gt 1 ]; then ((nvme_count--)) || true nvme_unmount_all $nvme_count - nvme_bind_all_in_order $nvme_count + nvme_bind_all_in_order nvme_recreate_namespace $nvme_count nvme_reserve_2_disk_per_numa $nvme_count @@ -171,9 +179,16 @@ if [ "$nvme_count" -gt 1 ]; then # cat /sys/class/nvme/nvme$i/address 2>/dev/null || echo "not found" #done - pushd /usr/share/daos/spdk/scripts/ - export PCI_ALLOWED="$SPDK_PCI_ALLOWED" - sudo ./setup.sh + if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then + export PCI_ALLOWED="$SPDK_PCI_ALLOWED" + pushd /usr/share/daos/spdk/scripts/ + set +e + sudo ./setup.sh + set -e + popd + else + echo "Required spdk/scripts/setup.sh not found!" + fi daos_server nvme scan -fi \ No newline at end of file +fi From e07c2773126ea3b05c4bbec72fea866d38f22808 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 Dec 2025 13:21:22 +0000 Subject: [PATCH 29/40] Ignore leap15 builds Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index be4102c785b..20e776bd705 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -632,6 +632,7 @@ pipeline { } } } +/* stage('Build on Leap 15.5') { when { beforeAgent true @@ -722,7 +723,9 @@ pipeline { } } } +*/ } + } stage('Unit Tests') { when { From 0688580603456c10d615e7c9bf600e7905e794af Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 18 Dec 2025 15:50:38 +0000 Subject: [PATCH 30/40] Fix - try to rollback few changes to reproduce the problem Cancel-prev-build: false Quick-build: true Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-test: true Skip-unit-tests: true Skip-func-vm: true Skip-fault-injection-test: true Skip-test-rpms: true Test-tag: test_recycle_pools_hw Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 7cdf9fc3f1e..fa51d1413d8 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -216,24 +216,17 @@ function nvme_reserve_2_disk_per_numa { fi fi sudo mkfs.ext4 -F "${dev}" -# if [ ! -d "${mnt}" ]; then -# sudo mkdir -p "${mnt}" -# fi -# sudo mount "${dev}" "${mnt}" + if [ ! -d "${mnt}" ]; then + sudo mkdir -p "${mnt}" + fi + sudo mount "${dev}" "${mnt}" done SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space } -nvme_count=$(nvme_count_devices) -if [ "$nvme_count" -gt 1 ]; then - ((nvme_count--)) || true - nvme_unmount_all $nvme_count - nvme_bind_all_in_order - nvme_recreate_namespace $nvme_count - nvme_reserve_2_disk_per_numa $nvme_count - +function setup_spdk_nvme { if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then - export PCI_ALLOWED="$SPDK_PCI_ALLOWED" + export PCI_ALLOWED="$1" pushd /usr/share/daos/spdk/scripts/ set +e sudo ./setup.sh @@ -242,6 +235,17 @@ if [ "$nvme_count" -gt 1 ]; then else echo "Required spdk/scripts/setup.sh not found!" fi +} + +nvme_count=$(nvme_count_devices) +if [ "$nvme_count" -gt 1 ]; then + ((nvme_count--)) || true + #nvme_unmount_all $nvme_count + nvme_bind_all_in_order + nvme_recreate_namespace $nvme_count + #nvme_reserve_2_disk_per_numa $nvme_count + #setup_spdk_nvme $SPDK_PCI_ALLOWED + fi # Workaround to enable binding devices back to nvme or vfio-pci after they are unbound from vfio-pci From 4752f2ba9e2f355df1dacca6d8251e679e590a22 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 30 Dec 2025 16:11:07 +0100 Subject: [PATCH 31/40] SRE-3150 ci: enable Functioanl HW Tests for el 9.6 TBD later Signed-off-by: Tomasz Gromadzki Cancel-prev-build: false Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 18 +- .../post_provision_config_nodes.sh | 369 ++++++++---------- src/tests/ftest/launch.py | 2 +- src/tests/ftest/pool/create_all_hw.py | 2 +- src/tests/ftest/pool/create_all_hw.yaml | 2 +- src/tests/ftest/util/pool_create_all_base.py | 2 +- 6 files changed, 170 insertions(+), 225 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 20e776bd705..6205a902a48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -252,7 +252,7 @@ pipeline { description: 'Type of build. Passed to scons as BUILD_TYPE. (I.e. dev, release, debug, etc.). ' + 'Defaults to release on an RC or dev otherwise.') string(name: 'TestRepeat', - defaultValue: '10', + defaultValue: '', description: 'Test-repeat to use for this run. Specifies the ' + 'number of times to repeat each functional test. ' + 'CAUTION: only use in combination with a reduced ' + @@ -303,7 +303,7 @@ pipeline { defaultValue: false, description: 'Do not build sources and RPMs on EL 9') booleanParam(name: 'CI_leap15_NOBUILD', - defaultValue: true, + defaultValue: false, description: 'Do not build sources and RPMs on Leap 15') booleanParam(name: 'CI_ALLOW_UNSTABLE_TEST', defaultValue: false, @@ -389,8 +389,7 @@ pipeline { defaultValue: 'ci_nlt_1', description: 'Label to use for NLT tests') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_LABEL', - defaultValue: 'ci_node-hdr-200_202-205X', - // defaultValue: 'ci_node-hdr-210_212-219X', + defaultValue: 'ci_nvme5', description: 'Label to use for the Functional Hardware Medium (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL', defaultValue: 'ci_ofed5', @@ -405,8 +404,8 @@ pipeline { defaultValue: 'ci_nvme9', description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION', - defaultValue: 'el9.5', - description: 'Label to use for 5 node Functional Hardware Medium (MD on SSD) stages') + defaultValue: 'el9.6', + description: 'Version of OS used in all Functional Hardware Medium stages') string(name: 'CI_STORAGE_PREP_LABEL', defaultValue: '', description: 'Label for cluster to do a DAOS Storage Preparation') @@ -632,7 +631,6 @@ pipeline { } } } -/* stage('Build on Leap 15.5') { when { beforeAgent true @@ -723,9 +721,7 @@ pipeline { } } } -*/ } - } stage('Unit Tests') { when { @@ -1181,6 +1177,7 @@ pipeline { /* groovylint-disable-next-line UnnecessaryGetter */ default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto', + image_version: params.FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION, run_if_pr: false, run_if_landing: false, job_status: job_status_internal @@ -1194,6 +1191,7 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto', provider: 'ofi+verbs;ofi_rxm', + image_version: params.FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION, run_if_pr: false, run_if_landing: false, job_status: job_status_internal @@ -1207,6 +1205,7 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto_md_on_ssd', provider: 'ofi+verbs;ofi_rxm', + image_version: params.FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION, run_if_pr: true, run_if_landing: false, job_status: job_status_internal @@ -1220,6 +1219,7 @@ pipeline { default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto', provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'), + image_version: params.FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION, run_if_pr: false, run_if_landing: false, job_status: job_status_internal diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index fa51d1413d8..bede9965fa4 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -5,7 +5,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent # -set -eux +set -euxo pipefail env > /root/last_run-env.txt @@ -69,93 +69,70 @@ if ! post_provision_config_nodes; then exit "$rc" fi +# Workaround to enable binding devices back to nvme or vfio-pci after they are unbound from vfio-pci +# to nvme. Sometimes the device gets unbound from vfio-pci, but it is not removed the iommu group +# for that device and future bindings to the device do not work, resulting in messages like, "NVMe +# SSD [xxxx:xx:xx.x] not found" when starting daos engines. +if lspci | grep -i nvme; then + export COVFILE=/tmp/test.cov + daos_server nvme reset && rmmod vfio_pci && modprobe vfio_pci +fi + # This workaround ensures that the NVMe configuration remains consistent across # all cluster nodes. # This prevents situations where the binding between NVMe devices and PCIe # addresses varies from restart to restart, resulting in error messages such as # "Failed to initialize SSD: [xxxx:xx:xx.x]' when DAOS engines are started. -NVME_MAX_GLOBAL=15 -function nvme_count_devices { - local count - count=$(lspci -D | grep -E 'Non-Volatile memory controller' | wc -l) - echo $count -} +SPDK_SETUP_CMD="/usr/share/daos/spdk/scripts/setup.sh" -function nvme_unmount_all { - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints." - set +e # we can try to unmount unexisted disks - for i in $(seq 0 $NVME_MAX); do - nvme="nvme${i}n1" - dev="/dev/${nvme}" - mnt="/mnt/${nvme}" - if [ -b "$dev" ]; then - mp=$(lsblk -nr -o MOUNTPOINT "$dev") - if [ -n "$mp" ]; then - echo "Unmounting $dev from $mp" - sudo umount -f "$dev"|| sudo umount -f "$mp" - elif [ -d "${mnt}" ]; then - echo "Unmounting ${mnt}" - sudo umount -f "${mnt}" - fi - elif [ -d "${mnt}" ]; then - echo "Force umount of ${mnt}" - sudo umount -f "${mnt}" - fi - rm -rf $mnt - done - set -e -} - -function nvme_bind_all_in_order { - # Find all PCI addresses for NVMe controllers - local nvme_pcie_addrs - nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') - - if [ -z "$nvme_pcie_addrs" ]; then - echo "No NVMe PCI devices found." +function check_spdk_setup_cmd { + if [ ! -d "$(dirname "$SPDK_SETUP_CMD")" ] || [ ! -f "$SPDK_SETUP_CMD" ]; then + echo -n "Required SPDK scripts directory $(dirname "$SPDK_SETUP_CMD")" + echo " or setup.sh not found!" return 1 fi + return 0 +} - #echo "Found PCI NVMe addresses:" - #echo "$nvme_pcie_addrs" - #echo +function get_nvme_count_devices { + lspci -D | grep -c -E "Non-Volatile memory controller" || true +} - for dir in /sys/class/nvme/*/; do - numa=$(cat ${dir}numa_node) - echo "$numa $dir: $(ls -la ${dir} | grep device | awk -F'-> ' '{print $2}' | sed 's|.*/||')" - done +function pci_dev_is_mounted { + local pci_dev="${1:?Usage: pci_dev_is_mounted }" + $SPDK_SETUP_CMD setup | grep "$pci_dev" | grep -q "mount@" +} - # Unbind all NVMe devices - echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." - set +e # it's ok if a device isn't bound to one of the drivers - for addr in $nvme_pcie_addrs; do - if [ -e "/sys/bus/pci/drivers/nvme/${addr}" ]; then - echo "Unbinding $addr from nvme" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind - fi - if [ -e "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then - echo "Unbinding $addr from vfio-pci" - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind - fi - done - set -e +function nvme_dev_has_data { + local nvme_dev="${1:-?Usage: nvme_dev_has_data }" + $SPDK_SETUP_CMD setup | grep -q "data@${nvme_dev}" +} - echo - # Bind all NVMe devices in order - echo "Binding NVMe devices to nvme driver in sorted order..." - set +e # for debug purpose - for addr in $(echo "$nvme_pcie_addrs" | sort); do - echo "Binding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind - done - set -e +function pci_dev_get_numa { + local pci_dev="${1:?Usage: pci_dev_get_numa }" + local pci_dev_numa_path="/sys/bus/pci/devices/${pci_dev}/numa_node" + cat "${pci_dev_numa_path}" } +function nvme_dev_get_first_by_pcie_addr() { + local pci_dev="${1:?Usage: nvme_dev_get_first_by_pcie_addr }" + local nvme_dir="/sys/bus/pci/devices/$pci_dev/nvme" + local nvme_dev + if [ -d "$nvme_dir" ]; then + for symlink in "$nvme_dir"/*; do + [ -e "$symlink" ] || continue + nvme_dev=$(basename "$symlink") + echo -n "${nvme_dev}" + return + done + else + echo "ERROR nvme_dev_get_first_by_pcie_addr can not find nvme for $pci_dev" + exit 1 + fi +} function nvme_recreate_namespace { - set +e # lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks # lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) # lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata @@ -167,166 +144,134 @@ function nvme_recreate_namespace { # ms = metadata size per block (0, 8, or 64 bytes). # rp = relative performance hint. - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - for i in $(seq 0 $NVME_MAX); do - dev="/dev/nvme${i}" - dev_ns="${dev}n1" - echo "Recreating namespace on $dev ..." - nvme delete-ns $dev -n 0x1 - nvme reset $dev - nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 - nvme attach-ns $dev -n 0x1 -c 0x41 - # selects LBA format index 0 (512BK) and no secure erase, just format. - nvme format $dev_ns --lbaf=0 --ses=0 --force - nvme reset $dev - nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" - done - set -e + local nvme_device="${1:?Usage: nvme_recreate_namespace }" + local nvme_dev_path="/dev/${nvme_device}" + local nvme_dev_ns_path="${nvme_dev_path}n1" + echo "Recreating namespace on $nvme_dev_path ..." + nvme delete-ns "$nvme_dev_path" -n 0x1 || \ + { echo "ERROR delete the ${nvme_dev_path} namespace failed"; exit 1; } + nvme reset "$nvme_dev_path" || \ + { echo "ERROR reset the ${nvme_dev_path} device failed"; exit 1; } + nvme create-ns "$nvme_dev_path" --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 || \ + { echo "ERROR create the ${nvme_dev_path} namespace failed"; exit 1; } + nvme attach-ns "$nvme_dev_path" -n 0x1 -c 0x41 || \ + { echo "ERROR attach the ${nvme_dev_path} namespace failed"; exit 1; } + # selects LBA format index 0 (512BK) and no secure erase, just format + nvme format "$nvme_dev_ns_path" --lbaf=0 --ses=0 --force || \ + { echo "ERROR format the ${nvme_dev_ns_path} namespace failed"; exit 1; } + nvme reset "$nvme_dev_path" || \ + { echo "ERROR reset the ${nvme_dev_path} namespace failed"; exit 1; } + nvme id-ns "$nvme_dev_ns_path" |grep -E "lbaf|nvmcap|nsze|ncap|nuse" + echo "${nvme_dev_ns_path} done" } -# FOR now limit to 2 devices per CPU NUMA node -: "${DAOS_CI_NVME_NUMA_LIMIT:=2}" - -SPDK_PCI_ALLOWED="" -function nvme_reserve_2_disk_per_numa { - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - local numa_node_0=0 - local numa_node_1=0 - for i in $(seq 0 $NVME_MAX); do - nvme="nvme${i}" - numa_path="/sys/class/nvme/${nvme}/numa_node" - dev="/dev/${nvme}n1" - mnt="/mnt/${nvme}n1" - numa_node="$(cat "$numa_path")" - pci_addr=$(basename "$(readlink -f /sys/class/nvme/${nvme}/device)") - echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" - if [ "$numa_node" -eq 0 ]; then - ((numa_node_0++)) || true - if [ "$numa_node_0" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then - SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " - echo "NUMA0: ${nvme} -> ${pci_addr}" - continue +# Format ext4 on each element of array after "daos_reserved" is reached. +function mkfs_on_nvme_over_limit { + local daos_reserved="${1:?Usage: mkfs_on_nvme_over_limit }" + shift + local nvme_devices=("$@") + local count=0 + local nvme_device nvme_device_ns_path + for nvme_device in "${nvme_devices[@]}"; do + if [ "$count" -ge "$daos_reserved" ]; then + nvme_device_ns_path="/dev/${nvme_device}n1" + if [ ! -e "$nvme_device_ns_path" ]; then + echo "INFO recreate namespace 1 on /dev/$nvme_device" + nvme_recreate_namespace "$nvme_device" + fi + if ! blkid -t TYPE=ext4 "$nvme_device_ns_path" >/dev/null 2>&1; then + echo "INFO mkfs on $nvme_device_ns_path" + sudo mkfs.ext4 -F "$nvme_device_ns_path" > /dev/null + else + echo "SKIP mkfs on $nvme_device_ns_path" fi else - ((numa_node_1++)) || true - if [ "$numa_node_1" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then - SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " - echo "NUMA1: ${nvme} -> ${pci_addr}" - continue + if nvme_dev_has_data "$nvme_device"; then + echo "INFO clean /dev/${nvme_device}" + nvme_recreate_namespace "$nvme_device" + else + echo "SKIP clean /dev/${nvme_device}" fi fi - sudo mkfs.ext4 -F "${dev}" - if [ ! -d "${mnt}" ]; then - sudo mkdir -p "${mnt}" - fi - sudo mount "${dev}" "${mnt}" + ((count++)) || true done - SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space } -function setup_spdk_nvme { - if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then - export PCI_ALLOWED="$1" - pushd /usr/share/daos/spdk/scripts/ - set +e - sudo ./setup.sh - set -e - popd +function nvme_setup { + local nvme_per_numa="${1:-?Usage: nvme_setup }" + local numa0_devices=() + local numa1_devices=() + local nvme_count + local nvme_dev nvme_pci_address numa_node + local nvme_pcie_address_all + local combined_numa_devices + + nvme_count=$(get_nvme_count_devices) + if [ "$nvme_count" -gt 1 ]; then + ((nvme_count--)) || true else - echo "Required spdk/scripts/setup.sh not found!" + return 0 + fi + + if ! check_spdk_setup_cmd; then + exit 1 fi -} - -nvme_count=$(nvme_count_devices) -if [ "$nvme_count" -gt 1 ]; then - ((nvme_count--)) || true - #nvme_unmount_all $nvme_count - nvme_bind_all_in_order - nvme_recreate_namespace $nvme_count - #nvme_reserve_2_disk_per_numa $nvme_count - #setup_spdk_nvme $SPDK_PCI_ALLOWED - -fi -# Workaround to enable binding devices back to nvme or vfio-pci after they are unbound from vfio-pci -# to nvme. Sometimes the device gets unbound from vfio-pci, but it is not removed the iommu group -# for that device and future bindings to the device do not work, resulting in messages like, "NVMe -# SSD [xxxx:xx:xx.x] not found" when starting daos engines. -if lspci | grep -i nvme; then - export COVFILE=/tmp/test.cov - daos_server nvme reset && rmmod vfio_pci && modprobe vfio_pci -fi + nvme_pcie_address_all=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}' | sort) -function mount_nvme_drive { - local drive="$1" - file_system=$(file -sL "/dev/$drive") - if [[ "$file_system" != *"ext4 filesystem"* ]]; then - yes | mkfs -t ext4 "/dev/$drive" + for nvme_pci_address in $nvme_pcie_address_all; do + # Skip already mounted namespace + if pci_dev_is_mounted "$nvme_pci_address"; then + echo "Skip already mounted namespace $nvme_pci_address" + continue fi - mkdir -p "/mnt/$drive" - mount "/dev/$drive" "/mnt/$drive" -} - - -nvme_class="/sys/class/nvme/" -function nvme_limit { - set +x - if [ ! -d "${nvme_class}" ] || [ -z "$(ls -A "${nvme_class}")" ]; then - echo "No NVMe devices found" - return + #echo "Binding $nvme_pci_address" + #echo "$nvme_pci_address" | sudo tee /sys/bus/pci/drivers/nvme/bind + nvme_dev=$(nvme_dev_get_first_by_pcie_addr "$nvme_pci_address") + numa_node="$(pci_dev_get_numa "$nvme_pci_address")" + if [ "$numa_node" -eq 0 ]; then + numa0_devices+=("$nvme_dev") + else + numa1_devices+=("$nvme_dev") fi - local numa0_devices=() - local numa1_devices=() - for nvme_path in "$nvme_class"*; do - nvme="$(basename "$nvme_path")n1" - numa_node="$(cat "${nvme_path}/numa_node")" - if mount | grep "$nvme"; then - continue - fi - if [ "$numa_node" -eq 0 ]; then - numa0_devices+=("$nvme") - else - numa1_devices+=("$nvme") - fi - done - echo numa0 "${numa0_devices[@]}" - echo numa1 "${numa1_devices[@]}" - if [ "${#numa0_devices[@]}" -gt 0 ] && [ "${#numa1_devices[@]}" -gt 0 ]; then - echo "balanced NVMe configuration possible" - nvme_count=0 - for nvme in "${numa0_devices[@]}"; do - if [ "$nvme_count" -ge "${DAOS_CI_NVME_NUMA_LIMIT}" ]; then - mount_nvme_drive "$nvme" - else - ((nvme_count++)) || true - fi - done - nvme_count=0 - for nvme in "${numa1_devices[@]}"; do - if [ "$nvme_count" -ge "${DAOS_CI_NVME_NUMA_LIMIT}" ]; then - mount_nvme_drive "$nvme" - else - ((nvme_count++)) || true - fi - done + done + echo NUMA0 PCIe devices: "${numa0_devices[@]}" + echo NUMA1 PCIe devices: "${numa1_devices[@]}" + if [ "${#numa0_devices[@]}" -ge "$nvme_per_numa" ] && \ + [ "${#numa1_devices[@]}" -ge "$nvme_per_numa" ]; then + echo "balanced NVMe configuration possible" + mkfs_on_nvme_over_limit "$nvme_per_numa" "${numa0_devices[@]}" + mkfs_on_nvme_over_limit "$nvme_per_numa" "${numa1_devices[@]}" + else + nvme_per_numa=$((nvme_per_numa + nvme_per_numa)) + combined_numa_devices=( "${numa0_devices[@]}" "${numa1_devices[@]}" ) + if [ "${#combined_numa_devices[@]}" -ge "$nvme_per_numa" ]; then + echo "balanced NVMe configuration not possible" + mkfs_on_nvme_over_limit "$nvme_per_numa" "${combined_numa_devices[@]}" else - echo "balanced NVMe configuration not possible" - for nvme in "${numa0_devices[@]}" "${numa1_devices[@]}"; do - ((needed = "$DAOS_CI_NVME_NUMA_LIMIT" + 1)) || true - nvme_count=0 - if [ "$nvme_count" -ge "$needed" ]; then - mount_nvme_drive "$nvme" - else - ((nvme_count++)) || true - fi - done + echo -n "ERROR not enough NVMe devices available." + echo -n " Expected at least ${nvme_per_numa}." + echo " Found ${#combined_numa_devices[@]}." fi - set -x + fi } -# Force only the desired number of NVMe devices to be seen by DAOS tests -# by mounting the extra ones. -# nvme_limit +function setup_spdk_nvme { + set +e + if check_spdk_setup_cmd; then + sudo "$SPDK_SETUP_CMD" status + fi + set -e +} + +#For now limit to 2 devices per CPU NUMA node +: "${DAOS_CI_NVME_NUMA_LIMIT:=2}" + +nvme_setup "$DAOS_CI_NVME_NUMA_LIMIT" +setup_spdk_nvme +if command -v daos_server >/dev/null 2>&1; then + daos_server nvme scan +fi systemctl enable nfs-server.service systemctl start nfs-server.service diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index e6cc07263e4..a62abe0a508 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -33,7 +33,7 @@ from util.yaml_utils import YamlException DEFAULT_LOGS_THRESHOLD = "2150M" # 2.1G -MAX_CI_REPETITIONS = 99 +MAX_CI_REPETITIONS = 10 class LaunchError(Exception): diff --git a/src/tests/ftest/pool/create_all_hw.py b/src/tests/ftest/pool/create_all_hw.py index 8b166439639..66f0cec8080 100644 --- a/src/tests/ftest/pool/create_all_hw.py +++ b/src/tests/ftest/pool/create_all_hw.py @@ -85,7 +85,7 @@ def test_recycle_pools_hw(self): pool_count = self.params.get("pool_count", "/run/test_recycle_pools_hw/*", 0) deltas_bytes = self.get_deltas("test_recycle_pools_hw") deltas_bytes[:] = [it * self.engines_count for it in deltas_bytes] - self.log.info("Test pool creation and destruction -- TESTING SAMIR --") + self.log.info("Test pool creation and destruction") self.log.info("\t- pool_count=%d", pool_count) self.log_deltas(*deltas_bytes) diff --git a/src/tests/ftest/pool/create_all_hw.yaml b/src/tests/ftest/pool/create_all_hw.yaml index 254dd930470..c4fcf5cb770 100644 --- a/src/tests/ftest/pool/create_all_hw.yaml +++ b/src/tests/ftest/pool/create_all_hw.yaml @@ -16,7 +16,7 @@ test_one_pool_hw: nvme: 1048576 # 1MiB test_recycle_pools_hw: - pool_count: 1 + pool_count: 50 deltas: scm: 18874368 # 18MiB nvme: 0 # 0MiB diff --git a/src/tests/ftest/util/pool_create_all_base.py b/src/tests/ftest/util/pool_create_all_base.py index f0f05f50ef4..660626c2c8e 100644 --- a/src/tests/ftest/util/pool_create_all_base.py +++ b/src/tests/ftest/util/pool_create_all_base.py @@ -226,7 +226,7 @@ def check_pool_recycling(self, pool_count, scm_delta_bytes, nvme_delta_bytes=Non first_pool_size = None for index in range(pool_count): self.log.info("Creating pool %d with all the available storage: size=100%%", index) - self.pool[index].size.update("90%", "pool[0].size") + self.pool[index].size.update("100%", "pool[0].size") self.pool[index].create() self.pool[index].get_info() s_total = self.pool[index].info.pi_space.ps_space.s_total From 5bc531a728a77efbe578ba9775b0c8a46eb5c868 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 31 Dec 2025 09:06:31 +0100 Subject: [PATCH 32/40] Script polishing Signed-off-by: Tomasz Gromadzki Cancel-prev-build: false Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Skip-func-test-vm-valgrind: true Skip-func-test-el8: true Skip-func-test-el9: true Skip-func-test-leap15: true Skip-fault-injection-test: true Skip-test-el-8.6-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-vmd: false Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 133 ++++++++---------- 1 file changed, 62 insertions(+), 71 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index bede9965fa4..56070f0dc38 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -83,7 +83,6 @@ fi # This prevents situations where the binding between NVMe devices and PCIe # addresses varies from restart to restart, resulting in error messages such as # "Failed to initialize SSD: [xxxx:xx:xx.x]' when DAOS engines are started. - SPDK_SETUP_CMD="/usr/share/daos/spdk/scripts/setup.sh" function check_spdk_setup_cmd { @@ -99,35 +98,35 @@ function get_nvme_count_devices { lspci -D | grep -c -E "Non-Volatile memory controller" || true } -function pci_dev_is_mounted { - local pci_dev="${1:?Usage: pci_dev_is_mounted }" - $SPDK_SETUP_CMD setup | grep "$pci_dev" | grep -q "mount@" +function pci_device_is_mounted { + local pci_device_address="${1:?Usage: pci_device_is_mounted }" + $SPDK_SETUP_CMD status 2>&1 | grep "$pci_device_address" | grep -q "mount@" } -function nvme_dev_has_data { - local nvme_dev="${1:-?Usage: nvme_dev_has_data }" - $SPDK_SETUP_CMD setup | grep -q "data@${nvme_dev}" +function pci_device_has_data { + local pci_device_address="${1:?Usage: pci_device_is_mounted }" + $SPDK_SETUP_CMD status 2>&1 | grep "$pci_device_address" | grep -q "data@" } -function pci_dev_get_numa { - local pci_dev="${1:?Usage: pci_dev_get_numa }" - local pci_dev_numa_path="/sys/bus/pci/devices/${pci_dev}/numa_node" - cat "${pci_dev_numa_path}" +function pci_device_get_numa { + local pci_device="${1:?Usage: pci_device_get_numa }" + local pci_device_numa_path="/sys/bus/pci/devices/${pci_device}/numa_node" + cat "${pci_device_numa_path}" } function nvme_dev_get_first_by_pcie_addr() { - local pci_dev="${1:?Usage: nvme_dev_get_first_by_pcie_addr }" - local nvme_dir="/sys/bus/pci/devices/$pci_dev/nvme" - local nvme_dev + local pci_device_address="${1:?Usage: nvme_dev_get_first_by_pcie_addr }" + local nvme_dir="/sys/bus/pci/devices/$pci_device_address/nvme" + local nvme_device symlink if [ -d "$nvme_dir" ]; then for symlink in "$nvme_dir"/*; do [ -e "$symlink" ] || continue - nvme_dev=$(basename "$symlink") - echo -n "${nvme_dev}" + nvme_device=$(basename "$symlink") + echo -n "${nvme_device}" return done else - echo "ERROR nvme_dev_get_first_by_pcie_addr can not find nvme for $pci_dev" + echo "ERROR nvme_dev_get_first_by_pcie_addr can not find nvme for $pci_device_address" exit 1 fi } @@ -145,38 +144,39 @@ function nvme_recreate_namespace { # rp = relative performance hint. local nvme_device="${1:?Usage: nvme_recreate_namespace }" - local nvme_dev_path="/dev/${nvme_device}" - local nvme_dev_ns_path="${nvme_dev_path}n1" - echo "Recreating namespace on $nvme_dev_path ..." - nvme delete-ns "$nvme_dev_path" -n 0x1 || \ - { echo "ERROR delete the ${nvme_dev_path} namespace failed"; exit 1; } - nvme reset "$nvme_dev_path" || \ - { echo "ERROR reset the ${nvme_dev_path} device failed"; exit 1; } - nvme create-ns "$nvme_dev_path" --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 || \ - { echo "ERROR create the ${nvme_dev_path} namespace failed"; exit 1; } - nvme attach-ns "$nvme_dev_path" -n 0x1 -c 0x41 || \ - { echo "ERROR attach the ${nvme_dev_path} namespace failed"; exit 1; } + local nvme_device_path="/dev/${nvme_device}" + local nvme_device_ns_path="${nvme_device_path}n1" + #echo "Recreating namespace on $nvme_device_path ..." + nvme delete-ns "$nvme_device_path" -n 0x1 || \ + { echo "ERROR delete the ${nvme_device_path} namespace failed"; exit 1; } + nvme reset "$nvme_device_path" || \ + { echo "ERROR reset the ${nvme_device_path} device failed"; exit 1; } + nvme create-ns "$nvme_device_path" --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 || \ + { echo "ERROR create the ${nvme_device_path} namespace failed"; exit 1; } + nvme attach-ns "$nvme_device_path" -n 0x1 -c 0x41 || \ + { echo "ERROR attach the ${nvme_device_path} namespace failed"; exit 1; } # selects LBA format index 0 (512BK) and no secure erase, just format - nvme format "$nvme_dev_ns_path" --lbaf=0 --ses=0 --force || \ - { echo "ERROR format the ${nvme_dev_ns_path} namespace failed"; exit 1; } - nvme reset "$nvme_dev_path" || \ - { echo "ERROR reset the ${nvme_dev_path} namespace failed"; exit 1; } - nvme id-ns "$nvme_dev_ns_path" |grep -E "lbaf|nvmcap|nsze|ncap|nuse" - echo "${nvme_dev_ns_path} done" + nvme format "$nvme_device_ns_path" --lbaf=0 --ses=0 --force || \ + { echo "ERROR format the ${nvme_device_ns_path} namespace failed"; exit 1; } + nvme reset "$nvme_device_path" || \ + { echo "ERROR reset the ${nvme_device_path} namespace failed"; exit 1; } + nvme id-ns "$nvme_device_ns_path" |grep -E "lbaf|nvmcap|nsze|ncap|nuse" + #echo "Recreating namespace on ${nvme_device_ns_path} done" } # Format ext4 on each element of array after "daos_reserved" is reached. function mkfs_on_nvme_over_limit { - local daos_reserved="${1:?Usage: mkfs_on_nvme_over_limit }" + local daos_nvme_numa_limit="${1:?Usage: mkfs_on_nvme_over_limit }" shift - local nvme_devices=("$@") + local nvme_pci_address_array=("$@") local count=0 - local nvme_device nvme_device_ns_path - for nvme_device in "${nvme_devices[@]}"; do - if [ "$count" -ge "$daos_reserved" ]; then + local nvme_pci_address nvme_device nvme_device_ns_path + for nvme_pci_address in "${nvme_pci_address_array[@]}"; do + nvme_device=$(nvme_dev_get_first_by_pcie_addr "$nvme_pci_address") + if [ "$count" -ge "$daos_nvme_numa_limit" ]; then nvme_device_ns_path="/dev/${nvme_device}n1" if [ ! -e "$nvme_device_ns_path" ]; then - echo "INFO recreate namespace 1 on /dev/$nvme_device" + echo "INFO recreate namespace 1 on /dev/${nvme_device} ${nvme_pci_address}" nvme_recreate_namespace "$nvme_device" fi if ! blkid -t TYPE=ext4 "$nvme_device_ns_path" >/dev/null 2>&1; then @@ -186,11 +186,11 @@ function mkfs_on_nvme_over_limit { echo "SKIP mkfs on $nvme_device_ns_path" fi else - if nvme_dev_has_data "$nvme_device"; then - echo "INFO clean /dev/${nvme_device}" + if pci_device_has_data "$nvme_pci_address"; then + echo "INFO clean /dev/${nvme_device} ${nvme_pci_address}" nvme_recreate_namespace "$nvme_device" else - echo "SKIP clean /dev/${nvme_device}" + echo "SKIP clean /dev/${nvme_device} ${nvme_pci_address}" fi fi ((count++)) || true @@ -198,13 +198,11 @@ function mkfs_on_nvme_over_limit { } function nvme_setup { - local nvme_per_numa="${1:-?Usage: nvme_setup }" - local numa0_devices=() - local numa1_devices=() - local nvme_count - local nvme_dev nvme_pci_address numa_node - local nvme_pcie_address_all - local combined_numa_devices + local daos_nvme_numa_limit="${1:-?Usage: nvme_setup }" + local numa0_pci_devices=() + local numa1_pci_devices=() + local all_numas_pci_devices + local nvme_count nvme_pcie_address_all nvme_pci_address numa_node nvme_count=$(get_nvme_count_devices) if [ "$nvme_count" -gt 1 ]; then @@ -221,38 +219,31 @@ function nvme_setup { for nvme_pci_address in $nvme_pcie_address_all; do # Skip already mounted namespace - if pci_dev_is_mounted "$nvme_pci_address"; then + if pci_device_is_mounted "$nvme_pci_address"; then echo "Skip already mounted namespace $nvme_pci_address" continue fi #echo "Binding $nvme_pci_address" #echo "$nvme_pci_address" | sudo tee /sys/bus/pci/drivers/nvme/bind - nvme_dev=$(nvme_dev_get_first_by_pcie_addr "$nvme_pci_address") - numa_node="$(pci_dev_get_numa "$nvme_pci_address")" + numa_node="$(pci_device_get_numa "$nvme_pci_address")" if [ "$numa_node" -eq 0 ]; then - numa0_devices+=("$nvme_dev") + numa0_pci_devices+=("$nvme_pci_address") else - numa1_devices+=("$nvme_dev") + numa1_pci_devices+=("$nvme_pci_address") fi done - echo NUMA0 PCIe devices: "${numa0_devices[@]}" - echo NUMA1 PCIe devices: "${numa1_devices[@]}" - if [ "${#numa0_devices[@]}" -ge "$nvme_per_numa" ] && \ - [ "${#numa1_devices[@]}" -ge "$nvme_per_numa" ]; then + echo NUMA0 PCIe devices: "${numa0_pci_devices[@]}" + echo NUMA1 PCIe devices: "${numa1_pci_devices[@]}" + if [ "${#numa0_pci_devices[@]}" -ge "$daos_nvme_numa_limit" ] && \ + [ "${#numa1_pci_devices[@]}" -ge "$daos_nvme_numa_limit" ]; then echo "balanced NVMe configuration possible" - mkfs_on_nvme_over_limit "$nvme_per_numa" "${numa0_devices[@]}" - mkfs_on_nvme_over_limit "$nvme_per_numa" "${numa1_devices[@]}" + mkfs_on_nvme_over_limit "$daos_nvme_numa_limit" "${numa0_pci_devices[@]}" + mkfs_on_nvme_over_limit "$daos_nvme_numa_limit" "${numa1_pci_devices[@]}" else - nvme_per_numa=$((nvme_per_numa + nvme_per_numa)) - combined_numa_devices=( "${numa0_devices[@]}" "${numa1_devices[@]}" ) - if [ "${#combined_numa_devices[@]}" -ge "$nvme_per_numa" ]; then - echo "balanced NVMe configuration not possible" - mkfs_on_nvme_over_limit "$nvme_per_numa" "${combined_numa_devices[@]}" - else - echo -n "ERROR not enough NVMe devices available." - echo -n " Expected at least ${nvme_per_numa}." - echo " Found ${#combined_numa_devices[@]}." - fi + daos_nvme_numa_limit=$((daos_nvme_numa_limit + daos_nvme_numa_limit)) + all_numas_pci_devices=( "${numa0_pci_devices[@]}" "${numa1_pci_devices[@]}" ) + echo "balanced NVMe configuration not possible" + mkfs_on_nvme_over_limit "$daos_nvme_numa_limit" "${all_numas_pci_devices[@]}" fi } From c22822537f6bd6ec92cc2a38c3d263d0f0531781 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 1 Jan 2026 19:32:04 +0100 Subject: [PATCH 33/40] FIx: calculate namespace size, wait for ns attachment Cancel-prev-build: false Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Skip-func-test-vm-valgrind: true Skip-func-test-el8: true Skip-func-test-el9: true Skip-func-test-leap15: true Skip-fault-injection-test: true Skip-test-el-8.6-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-vmd: false Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 65 +++++- src/tests/ftest/util/launch_utils.py | 3 +- utils/tools/nvme-full-init.sh | 194 ------------------ 3 files changed, 62 insertions(+), 200 deletions(-) delete mode 100644 utils/tools/nvme-full-init.sh diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 56070f0dc38..db2ddf7eecc 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -1,7 +1,7 @@ #!/bin/bash # # Copyright 2020-2023 Intel Corporation. -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # # SPDX-License-Identifier: BSD-2-Clause-Patent # @@ -131,6 +131,47 @@ function nvme_dev_get_first_by_pcie_addr() { fi } + +# Calculates --nsze and --ncap for a device so the namespace spans the full usable capacity +nvme_calc_full_nsze_ncap() { + local nvme_dev="${1:?Usage: nvme_calc_full_nsze_ncap }" + # Query the NVMe device info for total logical blocks and LBA size + local id_ctrl + id_ctrl=$(nvme id-ctrl "$nvme_dev") || { + echo "ERROR: nvme id-ctrl failed on $nvme_dev" + return 1 + } + + # Extract total NVM capacity in bytes (nvmcap) + # nvme-cli prints: "nvmcap : 4000884572160" + local nvmcap_bytes + nvmcap_bytes=$(echo "$id_ctrl" | awk -F: '/nvmcap/ {gsub(/ /, "", $2); print $2}') + if [[ -z "$nvmcap_bytes" ]]; then + echo "ERROR: Could not find nvmcap in nvme id-ctrl output" + return 1 + fi + + # Extract the size of a logical block (lba size), usually from nvme id-ns or id-ctrl + local lbads + local id_ns + id_ns=$(nvme id-ns "${nvme_dev}n1" 2>/dev/null || nvme id-ns "${nvme_dev}n1" 2>/dev/null) + if [[ -n "$id_ns" ]]; then + # Look for "lbads" line in id-ns output + lbads=$(echo "$id_ns" | awk -F: '/lbads/ {gsub(/ /, "", $2); print $2}') + fi + if [[ -z "$lbads" ]]; then + # fallback: Try to get LBA (logical block addressing) from id-ctrl if possible, else default to 512 + lbads=9 # Default for 512 bytes + fi + local lba_bytes=$((2 ** lbads)) + + # Calculate number of logical blocks + local lba_count=$(( nvmcap_bytes / lba_bytes )) + + # Output as hexadecimal format for nvme-cli + printf -- "--nsze=0x%x --ncap=0x%x\n" "$lba_count" "$lba_count" +} + function nvme_recreate_namespace { # lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks # lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) @@ -146,16 +187,29 @@ function nvme_recreate_namespace { local nvme_device="${1:?Usage: nvme_recreate_namespace }" local nvme_device_path="/dev/${nvme_device}" local nvme_device_ns_path="${nvme_device_path}n1" + local nvme_create_ns_opts #echo "Recreating namespace on $nvme_device_path ..." nvme delete-ns "$nvme_device_path" -n 0x1 || \ { echo "ERROR delete the ${nvme_device_path} namespace failed"; exit 1; } nvme reset "$nvme_device_path" || \ { echo "ERROR reset the ${nvme_device_path} device failed"; exit 1; } - nvme create-ns "$nvme_device_path" --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 || \ + nvme_create_ns_opts=$(nvme_calc_full_nsze_ncap "${nvme_device_path}") + nvme create-ns "$nvme_device_path" $nvme_create_ns_opts --flbas=0 || \ { echo "ERROR create the ${nvme_device_path} namespace failed"; exit 1; } nvme attach-ns "$nvme_device_path" -n 0x1 -c 0x41 || \ { echo "ERROR attach the ${nvme_device_path} namespace failed"; exit 1; } - # selects LBA format index 0 (512BK) and no secure erase, just format + # Wait up to 5 seconds for device node to appear + for i in {1..10}; do + if [ -b "$nvme_device_ns_path" ]; then + break + fi + sleep "$i" + done + if [ ! -b "$nvme_device_ns_path" ]; then + echo "ERROR: Namespace $nvme_device_ns_path did not appear after attach" + exit 1 + fi + # selects LBA format index 0 (512B) and no secure erase, just format nvme format "$nvme_device_ns_path" --lbaf=0 --ses=0 --force || \ { echo "ERROR format the ${nvme_device_ns_path} namespace failed"; exit 1; } nvme reset "$nvme_device_path" || \ @@ -247,7 +301,7 @@ function nvme_setup { fi } -function setup_spdk_nvme { +function spdk_setup_status { set +e if check_spdk_setup_cmd; then sudo "$SPDK_SETUP_CMD" status @@ -258,8 +312,9 @@ function setup_spdk_nvme { #For now limit to 2 devices per CPU NUMA node : "${DAOS_CI_NVME_NUMA_LIMIT:=2}" +spdk_setup_status nvme_setup "$DAOS_CI_NVME_NUMA_LIMIT" -setup_spdk_nvme +spdk_setup_status if command -v daos_server >/dev/null 2>&1; then daos_server nvme scan fi diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index f7477027358..ff6d090f194 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -1257,6 +1257,7 @@ def _setup_application_directory(self, logger, result): run_local(logger, f"ls -al '{self._test_env.app_dir}'") return 0 + # pylint: disable=too-many-locals def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop_daos, archive, rename, jenkins_log, core_files, threshold, user_create, code_coverage, job_results_dir, logdir, clear_mounts, cleanup_files): diff --git a/utils/tools/nvme-full-init.sh b/utils/tools/nvme-full-init.sh deleted file mode 100644 index 299cb7f31d7..00000000000 --- a/utils/tools/nvme-full-init.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -# -# Copyright 2025 Hewlett Packard Enterprise Development LP -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -# -# Full reinitialization of all NVMe devices on the DAOS cluster node -# - unmount all NVMe devices -# - unbind all NVMe devices and re-bind them to the nvme driver in sorted PCI address order. -# - create new logical block device (namespace) on each NVMe drive -# - mount all except two logical devices to build configuration required for DAOS cluster node -# - initialize two logical devices to work with SPDK (via vfio-pci driver) -# - verify configuration using the `daos_server nvme scan` command. -# -#set -e -#set -x -set -eu - -NVME_MAX_GLOBAL=15 -function nvme_count_devices { - local count - count=$(lspci -D | grep -E 'Non-Volatile memory controller' | wc -l) - echo $count -} - -function nvme_unmount_all { - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - echo "Unmounting all /dev/nvme[0-${NVME_MAX}]n1 mountpoints." - set +e # we can try to unmount unexisted disks - for i in $(seq 0 $NVME_MAX); do - nvme="nvme${i}n1" - dev="/dev/${nvme}" - mnt="/mnt/${nvme}" - if [ -b "$dev" ]; then - mp=$(lsblk -nr -o MOUNTPOINT "$dev") - if [ -n "$mp" ]; then - echo "Unmounting $dev from $mp" - sudo umount -f "$dev"|| sudo umount -f "$mp" - elif [ -d "${mnt}" ]; then - echo "Unmounting ${mnt}" - sudo umount -f "${mnt}" - fi - elif [ -d "${mnt}" ]; then - echo "Force umount of ${mnt}" - sudo umount -f "${mnt}" - fi - rm -rf $mnt - done - set -e -} - -function nvme_bind_all_in_order { - # Find all PCI addresses for NVMe controllers - local nvme_pcie_addrs - nvme_pcie_addrs=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}') - - if [ -z "$nvme_pcie_addrs" ]; then - echo "No NVMe PCI devices found." - return 1 - fi - - #echo "Found PCI NVMe addresses:" - #echo "$nvme_pcie_addrs" - #echo - - for dir in /sys/class/nvme/*/; do - numa=$(cat ${dir}numa_node) - echo "$numa $dir: $(ls -la ${dir} | grep device | awk -F'-> ' '{print $2}' | sed 's|.*/||')" - done - - # Unbind all NVMe devices - echo "Unbinding NVMe devices from nvme driver (or vfio-pci driver) ..." - set +e # it's ok if a device isn't bound to one of the drivers - for addr in $nvme_pcie_addrs; do - if [ -e "/sys/bus/pci/drivers/nvme/${addr}" ]; then - echo "Unbinding $addr from nvme" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/unbind - fi - if [ -e "/sys/bus/pci/drivers/vfio-pci/${addr}" ]; then - echo "Unbinding $addr from vfio-pci" - echo "$addr" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind - fi - done - set -e - - echo - # Bind all NVMe devices in order - echo "Binding NVMe devices to nvme driver in sorted order..." - set +e # for debug purpose - for addr in $(echo "$nvme_pcie_addrs" | sort); do - echo "Binding $addr" - echo "$addr" | sudo tee /sys/bus/pci/drivers/nvme/bind - done - set -e -} - - -function nvme_recreate_namespace { - set +e -# lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks -# lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) -# lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata -# lbaf 3 : ms:8 lbads:12 rp:0x2 → 4K + 8B metadata -# lbaf 4 : ms:64 lbads:12 rp:0x3 → 4K + 64B metadata -# lbads = log2(block size). -# 9 → 2⁹ = 512 bytes -# 12 → 2¹² = 4096 bytes (4K) -# ms = metadata size per block (0, 8, or 64 bytes). -# rp = relative performance hint. - - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - for i in $(seq 0 $NVME_MAX); do - dev="/dev/nvme${i}" - dev_ns="${dev}n1" - echo "Recreating namespace on $dev ..." - nvme delete-ns $dev -n 0x1 - nvme reset $dev - nvme create-ns $dev --nsze=0x1bf1f72b0 --ncap=0x1bf1f72b0 --flbas=0 - nvme attach-ns $dev -n 0x1 -c 0x41 - # selects LBA format index 0 (512BK) and no secure erase, just format. - nvme format $dev_ns --lbaf=0 --ses=0 --force - nvme reset $dev -# nvme id-ns $dev_ns |grep -E "lbaf|nvmcap|nsze|ncap|nuse" - done - set -e -} - -# FOR now limit to 2 devices per CPU NUMA node -: "${DAOS_CI_NVME_NUMA_LIMIT:=2}" - -SPDK_PCI_ALLOWED="" -function nvme_reserve_2_disk_per_numa { - local NVME_MAX=${1:-$NVME_MAX_GLOBAL} - local numa_node_0=0 - local numa_node_1=0 - for i in $(seq 0 $NVME_MAX); do - nvme="nvme${i}" - numa_path="/sys/class/nvme/${nvme}/numa_node" - dev="/dev/${nvme}n1" - mnt="/mnt/${nvme}n1" - numa_node="$(cat "$numa_path")" - pci_addr=$(basename "$(readlink -f /sys/class/nvme/${nvme}/device)") - echo "$nvme: NUMA node $numa_node, PCI addr $pci_addr, numa_node_0 $numa_node_0, numa_node_1 $numa_node_1" - if [ "$numa_node" -eq 0 ]; then - ((numa_node_0++)) || true - if [ "$numa_node_0" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then - SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " - echo "NUMA0: ${nvme} -> ${pci_addr}" - continue - fi - else - ((numa_node_1++)) || true - if [ "$numa_node_1" -le $DAOS_CI_NVME_NUMA_LIMIT ]; then - SPDK_PCI_ALLOWED="$SPDK_PCI_ALLOWED$pci_addr " - echo "NUMA1: ${nvme} -> ${pci_addr}" - continue - fi - fi - sudo mkfs.ext4 -F "${dev}" -# if [ ! -d "${mnt}" ]; then -# sudo mkdir -p "${mnt}" -# fi -# sudo mount "${dev}" "${mnt}" - done - SPDK_PCI_ALLOWED=${SPDK_PCI_ALLOWED% } # remove trailing space -} - -nvme_count=$(nvme_count_devices) -if [ "$nvme_count" -gt 1 ]; then - ((nvme_count--)) || true - nvme_unmount_all $nvme_count - nvme_bind_all_in_order - nvme_recreate_namespace $nvme_count - nvme_reserve_2_disk_per_numa $nvme_count - - #echo "Done. All NVMe devices have been re-bound in sorted PCI address order." - #for i in $(seq 0 $NVME_MAX_GLOBAL); do - # echo -n "nvme$i: " - # cat /sys/class/nvme/nvme$i/address 2>/dev/null || echo "not found" - #done - - if [ -d /usr/share/daos/spdk/scripts/ ] && [ -f /usr/share/daos/spdk/scripts/setup.sh ]; then - export PCI_ALLOWED="$SPDK_PCI_ALLOWED" - pushd /usr/share/daos/spdk/scripts/ - set +e - sudo ./setup.sh - set -e - popd - else - echo "Required spdk/scripts/setup.sh not found!" - fi - - daos_server nvme scan -fi From 10ea249a68b58a6af20d9f8b9831485cf6015a7a Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 1 Jan 2026 19:54:54 +0100 Subject: [PATCH 34/40] Fix: always recreate ns if does not exists Cancel-prev-build: false Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Skip-func-test-vm-valgrind: true Skip-func-test-el8: true Skip-func-test-el9: true Skip-func-test-leap15: true Skip-fault-injection-test: true Skip-test-el-8.6-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-vmd: false Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index db2ddf7eecc..1967dc9fb03 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -126,7 +126,7 @@ function nvme_dev_get_first_by_pcie_addr() { return done else - echo "ERROR nvme_dev_get_first_by_pcie_addr can not find nvme for $pci_device_address" + echo "ERROR: nvme_dev_get_first_by_pcie_addr can not find nvme for $pci_device_address" exit 1 fi } @@ -184,20 +184,30 @@ function nvme_recreate_namespace { # ms = metadata size per block (0, 8, or 64 bytes). # rp = relative performance hint. - local nvme_device="${1:?Usage: nvme_recreate_namespace }" + local nvme_device="${1:?Usage: nvme_recreate_namespace [skip_delete:true|false]}" + local skip_delete="${2:-false}" # true to skip, default false (delete enabled) local nvme_device_path="/dev/${nvme_device}" local nvme_device_ns_path="${nvme_device_path}n1" local nvme_create_ns_opts #echo "Recreating namespace on $nvme_device_path ..." + # Optionally skip delete step + if [[ "$skip_delete" != "true" ]]; then + nvme delete-ns "$nvme_device_path" -n 0x1 || \ + { echo "ERROR: delete the ${nvme_device_path} namespace failed"; exit 1; } + nvme reset "$nvme_device_path" || \ + { echo "ERROR: reset the ${nvme_device_path} device failed"; exit 1; } + else + echo "INFO: Skipping namespace delete on $nvme_device_path" + fi nvme delete-ns "$nvme_device_path" -n 0x1 || \ - { echo "ERROR delete the ${nvme_device_path} namespace failed"; exit 1; } + { echo "ERROR: delete the ${nvme_device_path} namespace failed"; exit 1; } nvme reset "$nvme_device_path" || \ - { echo "ERROR reset the ${nvme_device_path} device failed"; exit 1; } + { echo "ERROR: reset the ${nvme_device_path} device failed"; exit 1; } nvme_create_ns_opts=$(nvme_calc_full_nsze_ncap "${nvme_device_path}") nvme create-ns "$nvme_device_path" $nvme_create_ns_opts --flbas=0 || \ - { echo "ERROR create the ${nvme_device_path} namespace failed"; exit 1; } + { echo "ERROR: create the ${nvme_device_path} namespace failed"; exit 1; } nvme attach-ns "$nvme_device_path" -n 0x1 -c 0x41 || \ - { echo "ERROR attach the ${nvme_device_path} namespace failed"; exit 1; } + { echo "ERROR: attach the ${nvme_device_path} namespace failed"; exit 1; } # Wait up to 5 seconds for device node to appear for i in {1..10}; do if [ -b "$nvme_device_ns_path" ]; then @@ -211,9 +221,9 @@ function nvme_recreate_namespace { fi # selects LBA format index 0 (512B) and no secure erase, just format nvme format "$nvme_device_ns_path" --lbaf=0 --ses=0 --force || \ - { echo "ERROR format the ${nvme_device_ns_path} namespace failed"; exit 1; } + { echo "ERROR: format the ${nvme_device_ns_path} namespace failed"; exit 1; } nvme reset "$nvme_device_path" || \ - { echo "ERROR reset the ${nvme_device_path} namespace failed"; exit 1; } + { echo "ERROR: reset the ${nvme_device_path} namespace failed"; exit 1; } nvme id-ns "$nvme_device_ns_path" |grep -E "lbaf|nvmcap|nsze|ncap|nuse" #echo "Recreating namespace on ${nvme_device_ns_path} done" } @@ -227,12 +237,13 @@ function mkfs_on_nvme_over_limit { local nvme_pci_address nvme_device nvme_device_ns_path for nvme_pci_address in "${nvme_pci_address_array[@]}"; do nvme_device=$(nvme_dev_get_first_by_pcie_addr "$nvme_pci_address") + nvme_device_ns_path="/dev/${nvme_device}n1" + # always recreate namespace if it does not exist + if [ ! -e "$nvme_device_ns_path" ]; then + echo "INFO recreate namespace 1 on /dev/${nvme_device} ${nvme_pci_address}" + nvme_recreate_namespace "$nvme_device" true + fi if [ "$count" -ge "$daos_nvme_numa_limit" ]; then - nvme_device_ns_path="/dev/${nvme_device}n1" - if [ ! -e "$nvme_device_ns_path" ]; then - echo "INFO recreate namespace 1 on /dev/${nvme_device} ${nvme_pci_address}" - nvme_recreate_namespace "$nvme_device" - fi if ! blkid -t TYPE=ext4 "$nvme_device_ns_path" >/dev/null 2>&1; then echo "INFO mkfs on $nvme_device_ns_path" sudo mkfs.ext4 -F "$nvme_device_ns_path" > /dev/null From 3a45afebfa5cb2f7d47f6d7be15c7328371bd49d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 1 Jan 2026 20:21:05 +0100 Subject: [PATCH 35/40] Fix: calculate nvmcap_bytes based on tnvmcap Cancel-prev-build: false Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Skip-func-test-vm-valgrind: true Skip-func-test-el8: true Skip-func-test-el9: true Skip-func-test-leap15: true Skip-fault-injection-test: true Skip-test-el-8.6-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-vmd: false Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_nodes.sh | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 1967dc9fb03..65c4094e360 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -134,39 +134,37 @@ function nvme_dev_get_first_by_pcie_addr() { # Calculates --nsze and --ncap for a device so the namespace spans the full usable capacity nvme_calc_full_nsze_ncap() { - local nvme_dev="${1:?Usage: nvme_calc_full_nsze_ncap }" + local nvme_device="${1:?Usage: nvme_calc_full_nsze_ncap }" # Query the NVMe device info for total logical blocks and LBA size - local id_ctrl - id_ctrl=$(nvme id-ctrl "$nvme_dev") || { - echo "ERROR: nvme id-ctrl failed on $nvme_dev" - return 1 - } - - # Extract total NVM capacity in bytes (nvmcap) - # nvme-cli prints: "nvmcap : 4000884572160" + # Prefer tnvmcap, fallback to unvmcap if tnvmcap not found local nvmcap_bytes - nvmcap_bytes=$(echo "$id_ctrl" | awk -F: '/nvmcap/ {gsub(/ /, "", $2); print $2}') - if [[ -z "$nvmcap_bytes" ]]; then - echo "ERROR: Could not find nvmcap in nvme id-ctrl output" + nvmcap_bytes=$(nvme id-ctrl "$nvme_device" 2>/dev/null | \ + awk -F: ' + /tnvmcap/ {gsub(/[^0-9]/,"",$2); print $2; found=1; exit} + /unvmcap/ && !found {gsub(/[^0-9]/,"",$2); val=$2} + END{if(!found && val) print val} + ') + + if [[ -z "$nvmcap_bytes" || "$nvmcap_bytes" -eq 0 ]]; then + echo "ERROR: Could not find tnvmcap or unvmcap in nvme id-ctrl output" >&2 return 1 fi # Extract the size of a logical block (lba size), usually from nvme id-ns or id-ctrl - local lbads - local id_ns - id_ns=$(nvme id-ns "${nvme_dev}n1" 2>/dev/null || nvme id-ns "${nvme_dev}n1" 2>/dev/null) + local lbads id_ns lba_bytes lba_count + id_ns=$(nvme id-ns "${nvme_device}n1" 2>/dev/null || true) if [[ -n "$id_ns" ]]; then # Look for "lbads" line in id-ns output - lbads=$(echo "$id_ns" | awk -F: '/lbads/ {gsub(/ /, "", $2); print $2}') + lbads=$(echo "$id_ns" | awk -F: '/lbads/ {gsub(/[^0-9]/,"",$2); print $2; exit}') fi if [[ -z "$lbads" ]]; then # fallback: Try to get LBA (logical block addressing) from id-ctrl if possible, else default to 512 - lbads=9 # Default for 512 bytes + lbads=9 # Default for 512 bytes (2^9) fi - local lba_bytes=$((2 ** lbads)) + lba_bytes=$((2 ** lbads)) # Calculate number of logical blocks - local lba_count=$(( nvmcap_bytes / lba_bytes )) + lba_count=$(( nvmcap_bytes / lba_bytes )) # Output as hexadecimal format for nvme-cli printf -- "--nsze=0x%x --ncap=0x%x\n" "$lba_count" "$lba_count" From e45fd4301d1a47291b83e68fd55586db996487fc Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 1 Jan 2026 21:12:19 +0100 Subject: [PATCH 36/40] Trigger final validation Signed-off-by: Tomasz Gromadzki Cancel-prev-build: false Skip-fault-injection-test: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-md-on-ssd: false Skip-func-hw-test-medium-vmd: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-verbs-provider-md-on-ssd: false Skip-func-hw-test-medium-ucx-provider: false Skip-func-hw-test-large: false Skip-func-hw-test-large-md-on-ssd: false From e2f824e68488ddcb968c250963c04711f43e5dc9 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 1 Jan 2026 21:58:50 +0100 Subject: [PATCH 37/40] Fix: remove obsolete code Signed-off-by: Tomasz Gromadzki --- ci/provisioning/post_provision_config_nodes.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 65c4094e360..044f4622f89 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -268,9 +268,7 @@ function nvme_setup { local nvme_count nvme_pcie_address_all nvme_pci_address numa_node nvme_count=$(get_nvme_count_devices) - if [ "$nvme_count" -gt 1 ]; then - ((nvme_count--)) || true - else + if [ "$nvme_count" -le 1 ]; then return 0 fi From cd8846fd2d2ad11a32fdbb41f43b1e1e5cd510a1 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 2 Jan 2026 09:40:48 +0100 Subject: [PATCH 38/40] Fix: script optimization and bug fixing Cancel-prev-build: false Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Skip-func-test-vm-valgrind: true Skip-func-test-el8: true Skip-func-test-el9: true Skip-func-test-leap15: true Skip-fault-injection-test: true Skip-test-el-8.6-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-vmd: false Skip-func-hw-test-medium-md-on-ssd: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Skip-func-hw-test-large: false Skip-func-hw-test-large-md-on-ssd: false Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 4 +- .../post_provision_config_nodes.sh | 70 ++++++++++++------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6205a902a48..60e0b3c3796 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ /* groovylint-disable ParameterName, VariableName */ /* Copyright 2019-2024 Intel Corporation /* Copyright 2025 Google LLC - * Copyright 2025 Hewlett Packard Enterprise Development LP + * Copyright 2025-2026 Hewlett Packard Enterprise Development LP * All rights reserved. * * This file is part of the DAOS Project. It is subject to the license terms @@ -631,6 +631,7 @@ pipeline { } } } +/* stage('Build on Leap 15.5') { when { beforeAgent true @@ -721,6 +722,7 @@ pipeline { } } } +*/ } } stage('Unit Tests') { diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 044f4622f89..959b87257a3 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -85,7 +85,7 @@ fi # "Failed to initialize SSD: [xxxx:xx:xx.x]' when DAOS engines are started. SPDK_SETUP_CMD="/usr/share/daos/spdk/scripts/setup.sh" -function check_spdk_setup_cmd { +check_spdk_setup_cmd () { if [ ! -d "$(dirname "$SPDK_SETUP_CMD")" ] || [ ! -f "$SPDK_SETUP_CMD" ]; then echo -n "Required SPDK scripts directory $(dirname "$SPDK_SETUP_CMD")" echo " or setup.sh not found!" @@ -94,27 +94,47 @@ function check_spdk_setup_cmd { return 0 } -function get_nvme_count_devices { +get_nvme_count_devices () { lspci -D | grep -c -E "Non-Volatile memory controller" || true } -function pci_device_is_mounted { - local pci_device_address="${1:?Usage: pci_device_is_mounted }" - $SPDK_SETUP_CMD status 2>&1 | grep "$pci_device_address" | grep -q "mount@" +declare -A MOUNTED_PCI_DEVICES +declare -A PCI_DEVICES_WITH_DATA +pci_device_create_cache () { + MOUNTED_PCI_DEVICES=() + PCI_DEVICES_WITH_DATA=() + if check_spdk_setup_cmd; then + local status_output line pci_device_address + status_output="$($SPDK_SETUP_CMD status 2>&1)" + while read -r line; do + pci_device_address="${line%% *}" + if [[ "$pci_device_address" =~ ^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9]$ ]]; then + [[ "$line" == *"Active devices: mount@"* ]] && MOUNTED_PCI_DEVICES["$pci_device_address"]=1 + [[ "$line" == *"Active devices: data@"* ]] && PCI_DEVICES_WITH_DATA["$pci_device_address"]=1 + fi + done <<< "$status_output" + fi + echo "Cached ${#MOUNTED_PCI_DEVICES[@]} mounted PCI devices" + echo "Cached ${#PCI_DEVICES_WITH_DATA[@]} PCI devices with data" +} + +pci_device_is_mounted() { + local pci_device_address="${1:?Usage: pci_device_is_mounted }" + [[ -v MOUNTED_PCI_DEVICES[$pci_device_address] ]] } -function pci_device_has_data { - local pci_device_address="${1:?Usage: pci_device_is_mounted }" - $SPDK_SETUP_CMD status 2>&1 | grep "$pci_device_address" | grep -q "data@" +pci_device_has_data() { + local pci_device_address="${1:?Usage: pci_device_has_data }" + [[ -v PCI_DEVICES_WITH_DATA[$pci_device_address] ]] } -function pci_device_get_numa { +pci_device_get_numa () { local pci_device="${1:?Usage: pci_device_get_numa }" local pci_device_numa_path="/sys/bus/pci/devices/${pci_device}/numa_node" cat "${pci_device_numa_path}" } -function nvme_dev_get_first_by_pcie_addr() { +nvme_dev_get_first_by_pcie_addr (){ local pci_device_address="${1:?Usage: nvme_dev_get_first_by_pcie_addr }" local nvme_dir="/sys/bus/pci/devices/$pci_device_address/nvme" local nvme_device symlink @@ -131,9 +151,8 @@ function nvme_dev_get_first_by_pcie_addr() { fi } - # Calculates --nsze and --ncap for a device so the namespace spans the full usable capacity -nvme_calc_full_nsze_ncap() { +nvme_calc_full_nsze_ncap () { local nvme_device="${1:?Usage: nvme_calc_full_nsze_ncap }" # Query the NVMe device info for total logical blocks and LBA size # Prefer tnvmcap, fallback to unvmcap if tnvmcap not found @@ -151,7 +170,7 @@ nvme_calc_full_nsze_ncap() { fi # Extract the size of a logical block (lba size), usually from nvme id-ns or id-ctrl - local lbads id_ns lba_bytes lba_count + local lbads="" id_ns="" lba_bytes="" lba_count="" id_ns=$(nvme id-ns "${nvme_device}n1" 2>/dev/null || true) if [[ -n "$id_ns" ]]; then # Look for "lbads" line in id-ns output @@ -159,7 +178,7 @@ nvme_calc_full_nsze_ncap() { fi if [[ -z "$lbads" ]]; then # fallback: Try to get LBA (logical block addressing) from id-ctrl if possible, else default to 512 - lbads=9 # Default for 512 bytes (2^9) + lbads=12 # Default for 4096 bytes (2^12 = 4096) fi lba_bytes=$((2 ** lbads)) @@ -170,7 +189,7 @@ nvme_calc_full_nsze_ncap() { printf -- "--nsze=0x%x --ncap=0x%x\n" "$lba_count" "$lba_count" } -function nvme_recreate_namespace { +nvme_recreate_namespace (){ # lbaf 0 : ms:0 lbads:9 rp:0x1 (in use) → 512B blocks # lbaf 1 : ms:0 lbads:12 rp:0 → 4096B blocks (4K) # lbaf 2 : ms:8 lbads:9 rp:0x3 → 512B + 8B metadata @@ -187,7 +206,6 @@ function nvme_recreate_namespace { local nvme_device_path="/dev/${nvme_device}" local nvme_device_ns_path="${nvme_device_path}n1" local nvme_create_ns_opts - #echo "Recreating namespace on $nvme_device_path ..." # Optionally skip delete step if [[ "$skip_delete" != "true" ]]; then nvme delete-ns "$nvme_device_path" -n 0x1 || \ @@ -197,17 +215,16 @@ function nvme_recreate_namespace { else echo "INFO: Skipping namespace delete on $nvme_device_path" fi - nvme delete-ns "$nvme_device_path" -n 0x1 || \ - { echo "ERROR: delete the ${nvme_device_path} namespace failed"; exit 1; } nvme reset "$nvme_device_path" || \ { echo "ERROR: reset the ${nvme_device_path} device failed"; exit 1; } + nvme_create_ns_opts=$(nvme_calc_full_nsze_ncap "${nvme_device_path}") nvme create-ns "$nvme_device_path" $nvme_create_ns_opts --flbas=0 || \ { echo "ERROR: create the ${nvme_device_path} namespace failed"; exit 1; } nvme attach-ns "$nvme_device_path" -n 0x1 -c 0x41 || \ { echo "ERROR: attach the ${nvme_device_path} namespace failed"; exit 1; } # Wait up to 5 seconds for device node to appear - for i in {1..10}; do + for i in {1..5}; do if [ -b "$nvme_device_ns_path" ]; then break fi @@ -223,11 +240,10 @@ function nvme_recreate_namespace { nvme reset "$nvme_device_path" || \ { echo "ERROR: reset the ${nvme_device_path} namespace failed"; exit 1; } nvme id-ns "$nvme_device_ns_path" |grep -E "lbaf|nvmcap|nsze|ncap|nuse" - #echo "Recreating namespace on ${nvme_device_ns_path} done" } # Format ext4 on each element of array after "daos_reserved" is reached. -function mkfs_on_nvme_over_limit { +mkfs_on_nvme_over_limit () { local daos_nvme_numa_limit="${1:?Usage: mkfs_on_nvme_over_limit }" shift local nvme_pci_address_array=("$@") @@ -260,7 +276,7 @@ function mkfs_on_nvme_over_limit { done } -function nvme_setup { +nvme_setup (){ local daos_nvme_numa_limit="${1:-?Usage: nvme_setup }" local numa0_pci_devices=() local numa1_pci_devices=() @@ -268,7 +284,7 @@ function nvme_setup { local nvme_count nvme_pcie_address_all nvme_pci_address numa_node nvme_count=$(get_nvme_count_devices) - if [ "$nvme_count" -le 1 ]; then + if [ "$nvme_count" -le 1 ]; then # Expect at least 2 NVMe devices for proper setup return 0 fi @@ -276,6 +292,10 @@ function nvme_setup { exit 1 fi + set +x + pci_device_create_cache + set -x + nvme_pcie_address_all=$(lspci -D | awk '/Non-Volatile memory controller/{print $1}' | sort) for nvme_pci_address in $nvme_pcie_address_all; do @@ -284,8 +304,6 @@ function nvme_setup { echo "Skip already mounted namespace $nvme_pci_address" continue fi - #echo "Binding $nvme_pci_address" - #echo "$nvme_pci_address" | sudo tee /sys/bus/pci/drivers/nvme/bind numa_node="$(pci_device_get_numa "$nvme_pci_address")" if [ "$numa_node" -eq 0 ]; then numa0_pci_devices+=("$nvme_pci_address") @@ -311,7 +329,7 @@ function nvme_setup { function spdk_setup_status { set +e if check_spdk_setup_cmd; then - sudo "$SPDK_SETUP_CMD" status + "$SPDK_SETUP_CMD" status fi set -e } From 3325b1e9a91e18efa81ef599c1ea5e57c80e362b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 2 Jan 2026 21:58:43 +0100 Subject: [PATCH 39/40] Fix: use custom ctrl id Cancel-prev-build: false Priority: 2 Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Skip-func-test-vm-valgrind: true Skip-func-test-el8: true Skip-func-test-el9: true Skip-func-test-leap15: true Skip-fault-injection-test: true Skip-test-el-8.6-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-vmd: false Skip-func-hw-test-medium-md-on-ssd: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Skip-func-hw-test-large: false Skip-func-hw-test-large-md-on-ssd: false Signed-off-by: Tomasz Gromadzki --- ci/provisioning/post_provision_config_nodes.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 959b87257a3..bd3f52d8cae 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -151,8 +151,8 @@ nvme_dev_get_first_by_pcie_addr (){ fi } -# Calculates --nsze and --ncap for a device so the namespace spans the full usable capacity -nvme_calc_full_nsze_ncap () { +# Calculates --nsze for a device so the namespace spans the full usable capacity +nvme_calc_full_nsze () { local nvme_device="${1:?Usage: nvme_calc_full_nsze_ncap }" # Query the NVMe device info for total logical blocks and LBA size # Prefer tnvmcap, fallback to unvmcap if tnvmcap not found @@ -186,7 +186,7 @@ nvme_calc_full_nsze_ncap () { lba_count=$(( nvmcap_bytes / lba_bytes )) # Output as hexadecimal format for nvme-cli - printf -- "--nsze=0x%x --ncap=0x%x\n" "$lba_count" "$lba_count" + printf -- "0x%x\n" "$lba_count" } nvme_recreate_namespace (){ @@ -205,10 +205,10 @@ nvme_recreate_namespace (){ local skip_delete="${2:-false}" # true to skip, default false (delete enabled) local nvme_device_path="/dev/${nvme_device}" local nvme_device_ns_path="${nvme_device_path}n1" - local nvme_create_ns_opts + local nvme_nsze nvme_cntlid # Optionally skip delete step if [[ "$skip_delete" != "true" ]]; then - nvme delete-ns "$nvme_device_path" -n 0x1 || \ + nvme delete-ns "$nvme_device_path" -n 1 || \ { echo "ERROR: delete the ${nvme_device_path} namespace failed"; exit 1; } nvme reset "$nvme_device_path" || \ { echo "ERROR: reset the ${nvme_device_path} device failed"; exit 1; } @@ -218,10 +218,11 @@ nvme_recreate_namespace (){ nvme reset "$nvme_device_path" || \ { echo "ERROR: reset the ${nvme_device_path} device failed"; exit 1; } - nvme_create_ns_opts=$(nvme_calc_full_nsze_ncap "${nvme_device_path}") - nvme create-ns "$nvme_device_path" $nvme_create_ns_opts --flbas=0 || \ + nvme_nsze=$(nvme_calc_full_nsze "${nvme_device_path}") + nvme create-ns "$nvme_device_path" "--nsze=${nvme_nsze}" "--ncap=${nvme_nsze}" --flbas=0 || \ { echo "ERROR: create the ${nvme_device_path} namespace failed"; exit 1; } - nvme attach-ns "$nvme_device_path" -n 0x1 -c 0x41 || \ + nvme_cntlid=$(nvme id-ctrl "$nvme_device_path" | grep -iw cntlid | cut -d: -f2 | tr -d ' ') + nvme attach-ns "$nvme_device_path" -n 1 -c "$nvme_cntlid" || \ { echo "ERROR: attach the ${nvme_device_path} namespace failed"; exit 1; } # Wait up to 5 seconds for device node to appear for i in {1..5}; do From 91cff6f519c6895223ad5883a5ef75ce6cff07c6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 2 Jan 2026 23:56:03 +0100 Subject: [PATCH 40/40] Kick build on el8.8 Cancel-prev-build: false Priority: 2 Skip-build-leap15-gcc: true Skip-build-leap15-icc: true Skip-unit-tests:true Skip-unit-test: true Skip-NLT: true Skip-unit-test-memcheck: true Allow-unstable-test: true Skip-func-vm: true Skip-func-test-vm-valgrind: true Skip-func-test-el8: true Skip-func-test-el9: true Skip-func-test-leap15: true Skip-fault-injection-test: true Skip-test-el-8.6-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-vmd: false Skip-func-hw-test-medium-md-on-ssd: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Skip-func-hw-test-large: false Skip-func-hw-test-large-md-on-ssd: false Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 60e0b3c3796..7730f2a0d93 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -404,7 +404,7 @@ pipeline { defaultValue: 'ci_nvme9', description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') string(name: 'FUNCTIONAL_HARDWARE_MEDIUM_IMAGE_VERSION', - defaultValue: 'el9.6', + defaultValue: 'el8.8', description: 'Version of OS used in all Functional Hardware Medium stages') string(name: 'CI_STORAGE_PREP_LABEL', defaultValue: '',