From 3129c6e18d56119851be5faec107a5eaff2a1a19 Mon Sep 17 00:00:00 2001 From: Zdenek Kraus Date: Wed, 11 Mar 2026 17:33:05 +0100 Subject: [PATCH 1/6] feat(conftest): extended reporting for rptool --- Makefile | 12 ++++++++++-- testsuite/tests/conftest.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c829a32d..6cb77742 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ ifdef junit PYTEST += --junitxml=$(resultsdir)/junit-$(@F).xml -o junit_suite_name=$(@F) endif +# Collector PYTEST Override +collect: PYTEST = poetry run python -m pytest --tb=$(TB) --junitxml=$(resultsdir)/junit-00-$(@F).xml -o junit_suite_name=$(@F) + ifdef html PYTEST += --html=$(resultsdir)/report-$(@F).html --self-contained-html endif @@ -30,7 +33,7 @@ testsuite/%: FORCE poetry-no-dev test pytest tests singlecluster: kuadrant ## Run all single-cluster tests smoke: poetry-no-dev ## Run a small amount of selected tests to verify basic functionality - $(PYTEST) -n4 -m 'smoke' --dist loadfile --enforce $(flags) testsuite/tests/ + $(PYTEST) -n4 -m 'smoke' --dist loadfile --enforce $(flags) testsuite/tests/ || true kuadrant: poetry-no-dev ## Run all tests available on Kuadrant $(PYTEST) -n4 -m 'not standalone_only and not disruptive and not ui' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster @@ -42,7 +45,7 @@ authorino-standalone: poetry-no-dev ## Run only test capable of running with st $(PYTEST) -n4 -m 'authorino and not kuadrant_only and not disruptive' --dist loadfile --enforce --standalone $(flags) testsuite/tests/singlecluster/authorino/ limitador: poetry-no-dev ## Run only Limitador related tests - $(PYTEST) -n4 -m 'limitador and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/ + $(PYTEST) -n4 -m 'limitador and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/ || true dnstls: poetry-no-dev ## Run DNS and TLS tests $(PYTEST) -n4 -m '(dnspolicy or tlspolicy) and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/ @@ -77,6 +80,11 @@ coredns_two_primaries: poetry-no-dev ## Run coredns two primary tests $(PYTEST) -n1 -m 'coredns_two_primaries' --dist loadfile --enforce $(flags) testsuite/tests/multicluster/coredns/ +##@ Info Collection + +collect: poetry-no-dev + COLLECTOR_ENABLE=true $(PYTEST) testsuite/tests/info_collector.py + ##@ Misc commit-acceptance: black pylint mypy ## Runs pre-commit linting checks diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index efea536e..ebf7f249 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -4,6 +4,8 @@ from urllib.parse import urlparse import yaml +import os + import pytest from pytest_metadata.plugin import metadata_key # type: ignore from dynaconf import ValidationError @@ -113,7 +115,28 @@ def term_handler(): def pytest_collection_modifyitems(session, config, items): # pylint: disable=unused-argument - """Add cluster metadata to test items for ReportPortal integration.""" + """ + Add user properties to testcases for xml output + + Add cluster metadata to test items for ReportPortal integration. + + + This adds issue and issue-id properties to junit output, utilizes + pytest.mark.issue marker. + + This is copied from pytest examples to record custom properties in junit + https://docs.pytest.org/en/stable/usage.html + """ + + for item in items: + for marker in item.iter_markers(name="issue"): + issue = marker.args[0] + item.user_properties.append(("issue", issue)) + + ## extracting test's docstring for RP + item.user_properties.append(['__rp_case_description', item._obj.__doc__]) + + """""" try: collector = ReportPortalMetadataCollector() collector.collect_all_clusters() @@ -121,6 +144,14 @@ def pytest_collection_modifyitems(session, config, items): # pylint: disable=un except (OpenShiftPythonException, AttributeError, KeyError, ValidationError) as e: print(f"Warning: Component metadata collection failed: {e}") +def pytest_configure(config): + """Pytest post-execution configuration tuning""" + + ## Overriding junit_suite_name for collector + if os.environ.get('COLLECTOR_ENABLE'): + config.inicfg['junit_suite_name'] = 'info-collector' + # config.inicfg['junit_suite_name'] = junit_suite_name + @pytest.fixture(scope="session") def testconfig(): From 7718f7577277a864f5766f99db1413f3ec5d111c Mon Sep 17 00:00:00 2001 From: Zdenek Kraus Date: Wed, 11 Mar 2026 17:36:10 +0100 Subject: [PATCH 2/6] FIX: testcase docstring only if not None --- testsuite/tests/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index ebf7f249..a4198248 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -133,8 +133,9 @@ def pytest_collection_modifyitems(session, config, items): # pylint: disable=un issue = marker.args[0] item.user_properties.append(("issue", issue)) - ## extracting test's docstring for RP - item.user_properties.append(['__rp_case_description', item._obj.__doc__]) + ## extracting test's docstring for RP + if item._obj.__doc__: + item.user_properties.append(['__rp_case_description', item._obj.__doc__]) """""" try: From 522d6ee1dc3b2ccb8733bcbb29a141c463bc369c Mon Sep 17 00:00:00 2001 From: Zdenek Kraus Date: Wed, 11 Mar 2026 17:37:06 +0100 Subject: [PATCH 3/6] DOC: adding docstring to pytest_configure --- testsuite/tests/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index a4198248..3a977e26 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -146,9 +146,13 @@ def pytest_collection_modifyitems(session, config, items): # pylint: disable=un print(f"Warning: Component metadata collection failed: {e}") def pytest_configure(config): +<<<<<<< HEAD """Pytest post-execution configuration tuning""" +======= + """pytest config post-launch customization""" +>>>>>>> 173af15 (DOC: adding docstring to pytest_configure) - ## Overriding junit_suite_name for collector + ## Overriding junit_suite_name for collector only if os.environ.get('COLLECTOR_ENABLE'): config.inicfg['junit_suite_name'] = 'info-collector' # config.inicfg['junit_suite_name'] = junit_suite_name From 223b46e13afcf2f9078734b63c343bd978cd5867 Mon Sep 17 00:00:00 2001 From: Zdenek Kraus Date: Mon, 16 Mar 2026 15:18:06 +0100 Subject: [PATCH 4/6] NEW: tests/info_colelctor.py --- testsuite/tests/info_collector.py | 157 ++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 testsuite/tests/info_collector.py diff --git a/testsuite/tests/info_collector.py b/testsuite/tests/info_collector.py new file mode 100644 index 00000000..ad6d8e71 --- /dev/null +++ b/testsuite/tests/info_collector.py @@ -0,0 +1,157 @@ +""" +This module is not TRUE test. please do not rename to test_ + +It should only be executed in special mode defined by environment variable COLLECTOR_ENABLE=true +No xdist, only sequential. Supposed to be run before actual testsuite, isolated, to capture the values. + +example: +COLLECTOR_ENABLE=true pytest --junit-xml="${RESULTS_PATH}/junit_00_info_collector.xml" -o junit_logging=all testsuite/tests/info_collector.py +""" + + +import pytest +import os +import sys + +import random + +import subprocess + + +import logging +LOG = logging.getLogger(__name__) + +KUBECTL_ENABLE=True + +def kubectl(*args): + cmd = ['kubectl'] + cmd.extend(args) + result = None + try: + result = subprocess.check_output(cmd, text=True) + except Exception as e: + print(f'Failed with {e}') + return result + +def kube_image_version_parser(kubectl_output) -> list[tuple]: + result = [] + lines = kubectl_output.replace("'", "").split(sep=' ') + for line in lines: + print(f'{line}') + key, value = line.split(':') + key = key.split('/')[-1] + print(f'{key}:{value}') + result.append((key,value)) + return result + +@pytest.fixture(scope="session") +def properties_collector(record_testsuite_property): + """ Main property collector + + Any property added here, will be promoted to Launch level + + """ + # record_testsuite_property("os", random.choice(["ocp-4.20", 'ocp-4.19', 'ocp-4.18'])) + # record_testsuite_property("platform", random.choice(['onprem', 'aws', 'rosa', 'aro', 'aws-osd', 'gcp-osd', 'gcp'])) + # record_testsuite_property("version", random.choice(['1.3.0', '1.3.0-rc1', '1.2.0', '1.1.0', '1.0.0'])) + # record_testsuite_property("build", random.choice(['kuadrant', 'rhcl'])) + # record_testsuite_property("level", 'release') + + if KUBECTL_ENABLE: + kube_context = kubectl('config', 'current-context') + print(f'{kube_context=}') + if kube_context: + record_testsuite_property('kube_context', kube_context.strip()) + kuadrant_version = kubectl('get', '-n', 'kuadrant-system', 'pods', "-o=jsonpath='{.items[*].spec.containers[*].image}'") + if kuadrant_version: + print(f'{kuadrant_version}') + kuadrant_version_result = kube_image_version_parser(kuadrant_version) + for k, v in kuadrant_version_result: + record_testsuite_property(k, v) + + istio_version = kubectl('get', '-n', 'istio-system', 'pods', "-o=jsonpath='{.items[*].spec.containers[*].image}'") + if istio_version: + print(f'{istio_version}') + istio_version_result = kube_image_version_parser(istio_version) + for k,v in istio_version_result: + if 'sail-operator' not in k: + continue + record_testsuite_property(k,v) + + + +@pytest.fixture(scope="session") +def secondary_properties(record_testsuite_property): + """ Can be defined multiple times """ + record_testsuite_property('extra', 'secondary_properties') + +@pytest.fixture(scope="session") +def last_properties(record_testsuite_property): + """ Can be even done as teardown """ + yield + record_testsuite_property('last', 'property') + + + + +@pytest.fixture(scope="session") +def rp_suite_description(record_testsuite_property): + """ Ability to add something to suite description """ + suite_description=""" + # Not sure what to describe in test suite, but we can + """ + record_testsuite_property("__rp_suite_description", suite_description) + +@pytest.fixture(scope="session") +def rp_launch_description(record_testsuite_property): + """ Direct modification of RP Lauch description via promoted attribute + + description provided via commandline will be per-pended to this + """ + + + launch_description = """ + # Test Launch Description + + This is a sample description from the test pipeline + +**Cluster Information (2 clusters):** +- https://console.cluster1.example.com (cluster1) + - OCP: `4.18` + - Kuadrant: `quay.io/kuadrant/kuadrant-operator:v1.3.1` +- https://console.cluster2.example.com (cluster2) + - OCP: `4.20` + - Kuadrant: `quay.io/kuadrant/kuadrant-operator:nightly-latest` + """ + record_testsuite_property("__rp_launch_description", launch_description) + +@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled") +def test_collect(caplog, record_testsuite_property, properties_collector, secondary_properties, rp_launch_description, rp_suite_description): + '''Main collector entrypoint''' + + record_testsuite_property('collector', 'true') + # record_testsuite_property('__rp_dummy', 'this_should_not_be_there') + + LOG.info(f'Info message') + LOG.debug(f'Log Debug') + LOG.warning(f'warning message') + LOG.error(f'error message') + LOG.fatal(f'Fatal mesage') + + assert True + +@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled") +def test_kubectl(): + print + + +@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled") +def test_controller(): + + ## probably not worth it to collect uname from the pytest controller ? + print(f'{os.uname()=}') + # print(f'{os.=}') + ## probably not worth it, since it will be collecting launch arguments for collector only + print(f'{sys.argv=}') + + assert True \ No newline at end of file From 41e1bdcd15a70fed5f960f38f2c4e55cd19baef0 Mon Sep 17 00:00:00 2001 From: Zdenek Kraus Date: Mon, 16 Mar 2026 15:26:10 +0100 Subject: [PATCH 5/6] fixup! NEW: tests/info_colelctor.py --- testsuite/tests/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index 3a977e26..91c1689a 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -146,11 +146,7 @@ def pytest_collection_modifyitems(session, config, items): # pylint: disable=un print(f"Warning: Component metadata collection failed: {e}") def pytest_configure(config): -<<<<<<< HEAD """Pytest post-execution configuration tuning""" -======= - """pytest config post-launch customization""" ->>>>>>> 173af15 (DOC: adding docstring to pytest_configure) ## Overriding junit_suite_name for collector only if os.environ.get('COLLECTOR_ENABLE'): From 799eabf1855e6548ec264d6c4ea883cd17f637df Mon Sep 17 00:00:00 2001 From: Zdenek Kraus Date: Tue, 17 Mar 2026 09:21:34 +0100 Subject: [PATCH 6/6] NEW: initial integration of component_metadata.py to collector --- testsuite/component_metadata.py | 20 +++++++++------ testsuite/tests/conftest.py | 11 --------- testsuite/tests/info_collector.py | 41 ++++++++++++++++++++----------- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/testsuite/component_metadata.py b/testsuite/component_metadata.py index 82c5016d..51e8a91f 100644 --- a/testsuite/component_metadata.py +++ b/testsuite/component_metadata.py @@ -127,14 +127,18 @@ def _get_console_url(self, api_url): def _get_ocp_version(self, project): """Retrieve and format OCP version from cluster.""" - with project.context: - version_result = oc.selector("clusterversion").objects() - if version_result: - ocp_version = version_result[0].model.status.history[0].version - if ocp_version: - parts = ocp_version.split(".") - if len(parts) >= 2: - return f"{parts[0]}.{parts[1]}" + try: + with project.context: + version_result = oc.selector("clusterversion").objects() + if version_result: + ocp_version = version_result[0].model.status.history[0].version + if ocp_version: + parts = ocp_version.split(".") + if len(parts) >= 2: + return f"{parts[0]}.{parts[1]}" + except Exception as e: + logger.warning(str(e)) + return None def add_properties_to_items(self, items): diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index 91c1689a..859be981 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -118,9 +118,6 @@ def pytest_collection_modifyitems(session, config, items): # pylint: disable=un """ Add user properties to testcases for xml output - Add cluster metadata to test items for ReportPortal integration. - - This adds issue and issue-id properties to junit output, utilizes pytest.mark.issue marker. @@ -137,14 +134,6 @@ def pytest_collection_modifyitems(session, config, items): # pylint: disable=un if item._obj.__doc__: item.user_properties.append(['__rp_case_description', item._obj.__doc__]) - """""" - try: - collector = ReportPortalMetadataCollector() - collector.collect_all_clusters() - collector.add_properties_to_items(items) - except (OpenShiftPythonException, AttributeError, KeyError, ValidationError) as e: - print(f"Warning: Component metadata collection failed: {e}") - def pytest_configure(config): """Pytest post-execution configuration tuning""" diff --git a/testsuite/tests/info_collector.py b/testsuite/tests/info_collector.py index ad6d8e71..fd65f10d 100644 --- a/testsuite/tests/info_collector.py +++ b/testsuite/tests/info_collector.py @@ -18,6 +18,10 @@ import subprocess +from testsuite.component_metadata import ReportPortalMetadataCollector +from openshift_client import OpenShiftPythonException +from dynaconf import ValidationError + import logging LOG = logging.getLogger(__name__) @@ -80,6 +84,8 @@ def properties_collector(record_testsuite_property): + + @pytest.fixture(scope="session") def secondary_properties(record_testsuite_property): """ Can be defined multiple times """ @@ -102,28 +108,30 @@ def rp_suite_description(record_testsuite_property): """ record_testsuite_property("__rp_suite_description", suite_description) + +def get_cluster_information() -> str: + """Cluster information collector""" + cluster_info = "" + try: + collector = ReportPortalMetadataCollector() + collector.collect_all_clusters() + cluster_info = str(collector.all_cluster_metadata) + except (OpenShiftPythonException, AttributeError, KeyError, ValidationError) as e: + LOG.error("Component metadata collection failed: {e}") + print(f"Warning: Component metadata collection failed: {e}") + + return cluster_info + @pytest.fixture(scope="session") def rp_launch_description(record_testsuite_property): """ Direct modification of RP Lauch description via promoted attribute description provided via commandline will be per-pended to this """ + launch_description = get_cluster_information() + record_testsuite_property('__rp_launch_description', launch_description) - launch_description = """ - # Test Launch Description - - This is a sample description from the test pipeline - -**Cluster Information (2 clusters):** -- https://console.cluster1.example.com (cluster1) - - OCP: `4.18` - - Kuadrant: `quay.io/kuadrant/kuadrant-operator:v1.3.1` -- https://console.cluster2.example.com (cluster2) - - OCP: `4.20` - - Kuadrant: `quay.io/kuadrant/kuadrant-operator:nightly-latest` - """ - record_testsuite_property("__rp_launch_description", launch_description) @pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled") def test_collect(caplog, record_testsuite_property, properties_collector, secondary_properties, rp_launch_description, rp_suite_description): @@ -154,4 +162,7 @@ def test_controller(): ## probably not worth it, since it will be collecting launch arguments for collector only print(f'{sys.argv=}') - assert True \ No newline at end of file + assert True + + +