diff --git a/Makefile b/Makefile index c829a32d..6cb77742 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ ifdef junit PYTEST += --junitxml=$(resultsdir)/junit-$(@F).xml -o junit_suite_name=$(@F) endif +# Collector PYTEST Override +collect: PYTEST = poetry run python -m pytest --tb=$(TB) --junitxml=$(resultsdir)/junit-00-$(@F).xml -o junit_suite_name=$(@F) + ifdef html PYTEST += --html=$(resultsdir)/report-$(@F).html --self-contained-html endif @@ -30,7 +33,7 @@ testsuite/%: FORCE poetry-no-dev test pytest tests singlecluster: kuadrant ## Run all single-cluster tests smoke: poetry-no-dev ## Run a small amount of selected tests to verify basic functionality - $(PYTEST) -n4 -m 'smoke' --dist loadfile --enforce $(flags) testsuite/tests/ + $(PYTEST) -n4 -m 'smoke' --dist loadfile --enforce $(flags) testsuite/tests/ || true kuadrant: poetry-no-dev ## Run all tests available on Kuadrant $(PYTEST) -n4 -m 'not standalone_only and not disruptive and not ui' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster @@ -42,7 +45,7 @@ authorino-standalone: poetry-no-dev ## Run only test capable of running with st $(PYTEST) -n4 -m 'authorino and not kuadrant_only and not disruptive' --dist loadfile --enforce --standalone $(flags) testsuite/tests/singlecluster/authorino/ limitador: poetry-no-dev ## Run only Limitador related tests - $(PYTEST) -n4 -m 'limitador and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/ + $(PYTEST) -n4 -m 'limitador and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/ || true dnstls: poetry-no-dev ## Run DNS and TLS tests $(PYTEST) -n4 -m '(dnspolicy or tlspolicy) and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/singlecluster/ @@ -77,6 +80,11 @@ coredns_two_primaries: poetry-no-dev ## Run coredns two primary tests $(PYTEST) -n1 -m 'coredns_two_primaries' --dist loadfile --enforce $(flags) testsuite/tests/multicluster/coredns/ +##@ Info Collection + +collect: poetry-no-dev + COLLECTOR_ENABLE=true $(PYTEST) testsuite/tests/info_collector.py + ##@ Misc commit-acceptance: black pylint mypy ## Runs pre-commit linting checks diff --git a/testsuite/component_metadata.py b/testsuite/component_metadata.py index 82c5016d..51e8a91f 100644 --- a/testsuite/component_metadata.py +++ b/testsuite/component_metadata.py @@ -127,14 +127,18 @@ def _get_console_url(self, api_url): def _get_ocp_version(self, project): """Retrieve and format OCP version from cluster.""" - with project.context: - version_result = oc.selector("clusterversion").objects() - if version_result: - ocp_version = version_result[0].model.status.history[0].version - if ocp_version: - parts = ocp_version.split(".") - if len(parts) >= 2: - return f"{parts[0]}.{parts[1]}" + try: + with project.context: + version_result = oc.selector("clusterversion").objects() + if version_result: + ocp_version = version_result[0].model.status.history[0].version + if ocp_version: + parts = ocp_version.split(".") + if len(parts) >= 2: + return f"{parts[0]}.{parts[1]}" + except Exception as e: + logger.warning(str(e)) + return None def add_properties_to_items(self, items): diff --git a/testsuite/tests/conftest.py b/testsuite/tests/conftest.py index efea536e..859be981 100644 --- a/testsuite/tests/conftest.py +++ b/testsuite/tests/conftest.py @@ -4,6 +4,8 @@ from urllib.parse import urlparse import yaml +import os + import pytest from pytest_metadata.plugin import metadata_key # type: ignore from dynaconf import ValidationError @@ -113,13 +115,32 @@ def term_handler(): def pytest_collection_modifyitems(session, config, items): # pylint: disable=unused-argument - """Add cluster metadata to test items for ReportPortal integration.""" - try: - collector = ReportPortalMetadataCollector() - collector.collect_all_clusters() - collector.add_properties_to_items(items) - except (OpenShiftPythonException, AttributeError, KeyError, ValidationError) as e: - print(f"Warning: Component metadata collection failed: {e}") + """ + Add user properties to testcases for xml output + + This adds issue and issue-id properties to junit output, utilizes + pytest.mark.issue marker. + + This is copied from pytest examples to record custom properties in junit + https://docs.pytest.org/en/stable/usage.html + """ + + for item in items: + for marker in item.iter_markers(name="issue"): + issue = marker.args[0] + item.user_properties.append(("issue", issue)) + + ## extracting test's docstring for RP + if item._obj.__doc__: + item.user_properties.append(['__rp_case_description', item._obj.__doc__]) + +def pytest_configure(config): + """Pytest post-execution configuration tuning""" + + ## Overriding junit_suite_name for collector only + if os.environ.get('COLLECTOR_ENABLE'): + config.inicfg['junit_suite_name'] = 'info-collector' + # config.inicfg['junit_suite_name'] = junit_suite_name @pytest.fixture(scope="session") diff --git a/testsuite/tests/info_collector.py b/testsuite/tests/info_collector.py new file mode 100644 index 00000000..fd65f10d --- /dev/null +++ b/testsuite/tests/info_collector.py @@ -0,0 +1,168 @@ +""" +This module is not TRUE test. please do not rename to test_ + +It should only be executed in special mode defined by environment variable COLLECTOR_ENABLE=true +No xdist, only sequential. Supposed to be run before actual testsuite, isolated, to capture the values. + +example: +COLLECTOR_ENABLE=true pytest --junit-xml="${RESULTS_PATH}/junit_00_info_collector.xml" -o junit_logging=all testsuite/tests/info_collector.py +""" + + +import pytest +import os +import sys + +import random + +import subprocess + + +from testsuite.component_metadata import ReportPortalMetadataCollector +from openshift_client import OpenShiftPythonException +from dynaconf import ValidationError + +import logging +LOG = logging.getLogger(__name__) + +KUBECTL_ENABLE=True + +def kubectl(*args): + cmd = ['kubectl'] + cmd.extend(args) + result = None + try: + result = subprocess.check_output(cmd, text=True) + except Exception as e: + print(f'Failed with {e}') + return result + +def kube_image_version_parser(kubectl_output) -> list[tuple]: + result = [] + lines = kubectl_output.replace("'", "").split(sep=' ') + for line in lines: + print(f'{line}') + key, value = line.split(':') + key = key.split('/')[-1] + print(f'{key}:{value}') + result.append((key,value)) + return result + +@pytest.fixture(scope="session") +def properties_collector(record_testsuite_property): + """ Main property collector + + Any property added here, will be promoted to Launch level + + """ + # record_testsuite_property("os", random.choice(["ocp-4.20", 'ocp-4.19', 'ocp-4.18'])) + # record_testsuite_property("platform", random.choice(['onprem', 'aws', 'rosa', 'aro', 'aws-osd', 'gcp-osd', 'gcp'])) + # record_testsuite_property("version", random.choice(['1.3.0', '1.3.0-rc1', '1.2.0', '1.1.0', '1.0.0'])) + # record_testsuite_property("build", random.choice(['kuadrant', 'rhcl'])) + # record_testsuite_property("level", 'release') + + if KUBECTL_ENABLE: + kube_context = kubectl('config', 'current-context') + print(f'{kube_context=}') + if kube_context: + record_testsuite_property('kube_context', kube_context.strip()) + kuadrant_version = kubectl('get', '-n', 'kuadrant-system', 'pods', "-o=jsonpath='{.items[*].spec.containers[*].image}'") + if kuadrant_version: + print(f'{kuadrant_version}') + kuadrant_version_result = kube_image_version_parser(kuadrant_version) + for k, v in kuadrant_version_result: + record_testsuite_property(k, v) + + istio_version = kubectl('get', '-n', 'istio-system', 'pods', "-o=jsonpath='{.items[*].spec.containers[*].image}'") + if istio_version: + print(f'{istio_version}') + istio_version_result = kube_image_version_parser(istio_version) + for k,v in istio_version_result: + if 'sail-operator' not in k: + continue + record_testsuite_property(k,v) + + + + + +@pytest.fixture(scope="session") +def secondary_properties(record_testsuite_property): + """ Can be defined multiple times """ + record_testsuite_property('extra', 'secondary_properties') + +@pytest.fixture(scope="session") +def last_properties(record_testsuite_property): + """ Can be even done as teardown """ + yield + record_testsuite_property('last', 'property') + + + + +@pytest.fixture(scope="session") +def rp_suite_description(record_testsuite_property): + """ Ability to add something to suite description """ + suite_description=""" + # Not sure what to describe in test suite, but we can + """ + record_testsuite_property("__rp_suite_description", suite_description) + + +def get_cluster_information() -> str: + """Cluster information collector""" + cluster_info = "" + try: + collector = ReportPortalMetadataCollector() + collector.collect_all_clusters() + cluster_info = str(collector.all_cluster_metadata) + except (OpenShiftPythonException, AttributeError, KeyError, ValidationError) as e: + LOG.error("Component metadata collection failed: {e}") + print(f"Warning: Component metadata collection failed: {e}") + + return cluster_info + +@pytest.fixture(scope="session") +def rp_launch_description(record_testsuite_property): + """ Direct modification of RP Lauch description via promoted attribute + + description provided via commandline will be per-pended to this + """ + launch_description = get_cluster_information() + + record_testsuite_property('__rp_launch_description', launch_description) + + +@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled") +def test_collect(caplog, record_testsuite_property, properties_collector, secondary_properties, rp_launch_description, rp_suite_description): + '''Main collector entrypoint''' + + record_testsuite_property('collector', 'true') + # record_testsuite_property('__rp_dummy', 'this_should_not_be_there') + + LOG.info(f'Info message') + LOG.debug(f'Log Debug') + LOG.warning(f'warning message') + LOG.error(f'error message') + LOG.fatal(f'Fatal mesage') + + assert True + +@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled") +def test_kubectl(): + print + + +@pytest.mark.skipif(not os.environ.get('COLLECTOR_ENABLE'), reason="collector was not excplicitly enabled") +def test_controller(): + + ## probably not worth it to collect uname from the pytest controller ? + print(f'{os.uname()=}') + # print(f'{os.=}') + ## probably not worth it, since it will be collecting launch arguments for collector only + print(f'{sys.argv=}') + + assert True + + +