WIP: feat(testsuite): New reporting

Makefile

@@ -16,6 +16,9 @@ ifdef junit
 PYTEST += --junitxml=$(resultsdir)/junit-$(@F).xml -o junit_suite_name=$(@F)
 endif
 
+# Collector PYTEST Override
+collect: PYTEST = poetry run python -m pytest --tb=$(TB) --junitxml=$(resultsdir)/junit-00-$(@F).xml -o junit_suite_name=$(@F)
+
 ifdef html
 PYTEST += --html=$(resultsdir)/report-$(@F).html --self-contained-html
 endif
@@ -30,7 +33,7 @@ testsuite/%: FORCE poetry-no-dev
 test pytest tests singlecluster: kuadrant  ## Run all single-cluster tests
 
 smoke: poetry-no-dev  ## Run a small amount of selected tests to verify basic functionality
-	$(PYTEST) -n4 -m 'smoke' --dist loadfile --enforce $(flags) testsuite/tests/
+	$(PYTEST) -n4 -m 'smoke' --dist loadfile --enforce $(flags) testsuite/tests/ || true
 
 kuadrant: poetry-no-dev  ## Run all tests available on Kuadrant
 	$(PYTEST) -n4 -m 'not standalone_only and not disruptive and not ui' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster
@@ -42,7 +45,7 @@ authorino-standalone: poetry-no-dev  ## Run only test capable of running with st
 	$(PYTEST) -n4 -m 'authorino and not kuadrant_only and not disruptive' --dist loadfile --enforce --standalone $(flags) testsuite/tests/singlecluster/authorino/
 
 limitador: poetry-no-dev  ## Run only Limitador related tests
-	$(PYTEST) -n4 -m 'limitador and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/
+	$(PYTEST) -n4 -m 'limitador and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/ || true
 
 dnstls: poetry-no-dev  ## Run DNS and TLS tests
 	$(PYTEST) -n4 -m '(dnspolicy or tlspolicy) and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/
@@ -77,6 +80,11 @@ coredns_two_primaries: poetry-no-dev  ## Run coredns two primary tests
 	$(PYTEST) -n1 -m 'coredns_two_primaries' --dist loadfile --enforce $(flags) testsuite/tests/multicluster/coredns/
 
 
+##@ Info Collection
+
+collect: poetry-no-dev
+	COLLECTOR_ENABLE=true $(PYTEST) testsuite/tests/info_collector.py
+
 ##@ Misc
 
 commit-acceptance: black pylint mypy  ## Runs pre-commit linting checks

testsuite/component_metadata.py

@@ -127,14 +127,18 @@ def _get_console_url(self, api_url):
 
     def _get_ocp_version(self, project):
         """Retrieve and format OCP version from cluster."""
-        with project.context:
-            version_result = oc.selector("clusterversion").objects()
-            if version_result:
-                ocp_version = version_result[0].model.status.history[0].version
-                if ocp_version:
-                    parts = ocp_version.split(".")
-                    if len(parts) >= 2:
-                        return f"{parts[0]}.{parts[1]}"
+        try:
+            with project.context:
+                version_result = oc.selector("clusterversion").objects()
+                if version_result:
+                    ocp_version = version_result[0].model.status.history[0].version
+                    if ocp_version:
+                        parts = ocp_version.split(".")
+                        if len(parts) >= 2:
+                            return f"{parts[0]}.{parts[1]}"
+        except Exception as e:
+            logger.warning(str(e))
+
         return None
 
     def add_properties_to_items(self, items):

testsuite/tests/conftest.py

@@ -4,6 +4,8 @@
 from urllib.parse import urlparse
 import yaml
 
+import os
+
 import pytest
 from pytest_metadata.plugin import metadata_key  # type: ignore
 from dynaconf import ValidationError
@@ -113,13 +115,32 @@ def term_handler():
 
 
 def pytest_collection_modifyitems(session, config, items):  # pylint: disable=unused-argument
-    """Add cluster metadata to test items for ReportPortal integration."""
-    try:
-        collector = ReportPortalMetadataCollector()
-        collector.collect_all_clusters()
-        collector.add_properties_to_items(items)
-    except (OpenShiftPythonException, AttributeError, KeyError, ValidationError) as e:
-        print(f"Warning: Component metadata collection failed: {e}")
+    """
+    Add user properties to testcases for xml output
+
+    This adds issue and issue-id properties to junit output, utilizes
+    pytest.mark.issue marker.
+
+    This is copied from pytest examples to record custom properties in junit
+    https://docs.pytest.org/en/stable/usage.html
+    """
+
+    for item in items:
+        for marker in item.iter_markers(name="issue"):
+            issue = marker.args[0]
+            item.user_properties.append(("issue", issue))
+
+        ## extracting test's docstring for RP
+        if item._obj.__doc__:
+            item.user_properties.append(['__rp_case_description', item._obj.__doc__])
+
+def pytest_configure(config):
+    """Pytest post-execution configuration tuning"""
+
+    ## Overriding junit_suite_name for collector only
+    if os.environ.get('COLLECTOR_ENABLE'):
+        config.inicfg['junit_suite_name'] = 'info-collector'
+        # config.inicfg['junit_suite_name'] = junit_suite_name
 
 
 @pytest.fixture(scope="session")

testsuite/tests/info_collector.py

@@ -0,0 +1,168 @@
+"""
+This module is not TRUE test. please do not rename to test_
+
+It should only be executed in special mode defined by environment variable COLLECTOR_ENABLE=true
+No xdist, only sequential. Supposed to be run before actual testsuite, isolated, to capture the values.
+
+example:
+COLLECTOR_ENABLE=true pytest --junit-xml="${RESULTS_PATH}/junit_00_info_collector.xml" -o junit_logging=all testsuite/tests/info_collector.py 
+"""
+
+
+import pytest
+import os
+import sys
+
+import random
+
+import subprocess
+
+
+from testsuite.component_metadata import ReportPortalMetadataCollector
+from openshift_client import OpenShiftPythonException
+from dynaconf import ValidationError
+
+import logging
+LOG = logging.getLogger(__name__)
+
+KUBECTL_ENABLE=True
+
+def kubectl(*args):
+    cmd = ['kubectl']
+    cmd.extend(args)
+    result = None
+    try:
+        result = subprocess.check_output(cmd, text=True)
+    except Exception as e:
+        print(f'Failed with {e}')
+    return result
+
+def kube_image_version_parser(kubectl_output) -> list[tuple]:
+    result = []
+    lines = kubectl_output.replace("'", "").split(sep=' ')
+    for line in lines:
+        print(f'{line}')
+        key, value = line.split(':')
+        key = key.split('/')[-1]
+        print(f'{key}:{value}')
+        result.append((key,value))
+    return result
+
+@pytest.fixture(scope="session")
+def properties_collector(record_testsuite_property):
+    """ Main property collector
+
+    Any property added here, will be promoted to Launch level
+
+    """
+    # record_testsuite_property("os", random.choice(["ocp-4.20", 'ocp-4.19', 'ocp-4.18']))
+    # record_testsuite_property("platform", random.choice(['onprem', 'aws', 'rosa', 'aro', 'aws-osd', 'gcp-osd', 'gcp']))
+    # record_testsuite_property("version", random.choice(['1.3.0', '1.3.0-rc1', '1.2.0', '1.1.0', '1.0.0']))
+    # record_testsuite_property("build", random.choice(['kuadrant', 'rhcl']))
+    # record_testsuite_property("level", 'release')
+
+    if KUBECTL_ENABLE:
+        kube_context = kubectl('config', 'current-context')
+        print(f'{kube_context=}')
+        if kube_context:
+            record_testsuite_property('kube_context', kube_context.strip())
+        kuadrant_version = kubectl('get', '-n', 'kuadrant-system', 'pods', "-o=jsonpath='{.items[*].spec.containers[*].image}'")
+        if kuadrant_version:
+            print(f'{kuadrant_version}')
+            kuadrant_version_result = kube_image_version_parser(kuadrant_version)
+            for k, v in kuadrant_version_result:
+                record_testsuite_property(k, v)
+
+        istio_version = kubectl('get', '-n', 'istio-system', 'pods', "-o=jsonpath='{.items[*].spec.containers[*].image}'")
+        if istio_version:
+            print(f'{istio_version}')
+            istio_version_result = kube_image_version_parser(istio_version)
+            for k,v in istio_version_result:
+                if 'sail-operator' not in k:
+                    continue
+                record_testsuite_property(k,v)
+
+
+
+
+
+@pytest.fixture(scope="session")
+def secondary_properties(record_testsuite_property):
+    """ Can be defined multiple times """
+    record_testsuite_property('extra', 'secondary_properties')
+
+@pytest.fixture(scope="session")
+def last_properties(record_testsuite_property):
+    """ Can be even done as teardown """
+    yield
+    record_testsuite_property('last', 'property')
+
+
+
+
+@pytest.fixture(scope="session")
+def rp_suite_description(record_testsuite_property):
+    """ Ability to add something to suite description """
+    suite_description="""
+    # Not sure what to describe in test suite, but we can
+    """
+    record_testsuite_property("__rp_suite_description", suite_description)
+
+
+def get_cluster_information() -> str:
+    """Cluster information collector"""
+    cluster_info = ""
+    try:
+        collector = ReportPortalMetadataCollector()
+        collector.collect_all_clusters()
+        cluster_info = str(collector.all_cluster_metadata)
+    except (OpenShiftPythonException, AttributeError, KeyError, ValidationError) as e:
+        LOG.error("Component metadata collection failed: {e}")
+        print(f"Warning: Component metadata collection failed: {e}")
+
+    return cluster_info
+
+@pytest.fixture(scope="session")
+def rp_launch_description(record_testsuite_property):
+    """ Direct modification of RP Lauch description via promoted attribute
+
+    description provided via commandline will be per-pended to this
+    """
+    launch_description = get_cluster_information()
+
+    record_testsuite_property('__rp_launch_description', launch_description)
+
+
+@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled")
+def test_collect(caplog, record_testsuite_property, properties_collector, secondary_properties, rp_launch_description, rp_suite_description):
+    '''Main collector entrypoint'''
+
+    record_testsuite_property('collector', 'true')
+    # record_testsuite_property('__rp_dummy', 'this_should_not_be_there')
+
+    LOG.info(f'Info message')
+    LOG.debug(f'Log Debug')
+    LOG.warning(f'warning message')
+    LOG.error(f'error message')
+    LOG.fatal(f'Fatal mesage')
+
+    assert True
+
+@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled")
+def test_kubectl():
+    print
+
+
+@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled")
+def test_controller():
+
+    ## probably not worth it to collect uname from the pytest controller ?
+    print(f'{os.uname()=}')
+    # print(f'{os.=}')
+    ## probably not worth it, since it will be collecting launch arguments for collector only
+    print(f'{sys.argv=}')
+
+    assert True
+
+
+