1421: load variables ccode correctly

cdisc_rules_engine/dataset_builders/base_dataset_builder.py

@@ -9,7 +9,7 @@
     get_corresponding_datasets,
     tag_source,
 )
-from typing import List, Iterable
+from typing import List, Iterable, Optional
 from cdisc_rules_engine.utilities import sdtm_utilities
 from cdisc_rules_engine.utilities.rule_processor import RuleProcessor
 from cdisc_rules_engine.models.dataset.dataset_interface import DatasetInterface
@@ -156,9 +156,17 @@ def get_define_xml_variables_metadata(self) -> List[dict]:
         define_xml_reader = DefineXMLReaderFactory.get_define_xml_reader(
             self.dataset_path, self.define_xml_path, self.data_service, self.cache
         )
-        return define_xml_reader.extract_variables_metadata(
-            domain_name=self.dataset_metadata.domain
-        )
+        # If domain is not set and this is a SUPP domain, use rdomain
+        domain = self.dataset_metadata.domain
+        if not domain and getattr(self.dataset_metadata, "is_supp", False):
+            domain = getattr(self.dataset_metadata, "rdomain", None)
+            name = getattr(self.dataset_metadata, "name", None)
+            return define_xml_reader.extract_variables_metadata(
+                domain_name=domain, name=name
+            )
+        if not domain:
+            return []
+        return define_xml_reader.extract_variables_metadata(domain_name=domain)
 
     def get_define_xml_value_level_metadata(self) -> List[dict]:
         """
@@ -205,13 +213,16 @@ def get_library_variables_metadata(self) -> DatasetInterface:
         variables: List[dict] = sdtm_utilities.get_variables_metadata_from_standard(
             domain=domain, library_metadata=self.library_metadata
         )
+        variables_metadata: dict = self.library_metadata.variables_metadata.get(
+            domain, {}
+        )
         for variable in variables:
             variable["ccode"] = ""
-            if variable.get("codelistSubmissionValues"):
+            variable_metadata: Optional[dict] = variables_metadata.get(variable["name"])
+            if variable_metadata:
                 if "_links" in variable and "codelist" in variable["_links"]:
                     first_codelist = variable["_links"]["codelist"][0]
-                    href = first_codelist["href"]
-                    codelist_code = href.split("/")[-1]
+                    codelist_code = first_codelist["href"].split("/")[-1]
                     variable["ccode"] = codelist_code
             if "role" not in variable:
                 variable["role"] = ""

cdisc_rules_engine/rules_engine.py

@@ -335,8 +335,6 @@ def validate_rule(
         elif (
             rule.get("rule_type")
             == RuleTypes.VARIABLE_METADATA_CHECK_AGAINST_DEFINE.value
-            or rule.get("rule_type")
-            == RuleTypes.VARIABLE_METADATA_CHECK_AGAINST_DEFINE_XML_AND_LIBRARY.value
         ):
             self.rule_processor.add_comparator_to_rule_conditions(
                 rule, comparator=None, target_prefix="define_"

cdisc_rules_engine/services/define_xml/base_define_xml_reader.py

@@ -157,13 +157,15 @@ def extract_domain_metadata(self, domain_name: str = None) -> dict:
         return domain_metadata_dict
 
     @cached("define-variables-metadata")
-    def extract_variables_metadata(self, domain_name: str = None) -> List[dict]:
+    def extract_variables_metadata(
+        self, domain_name: str = None, name: str = None
+    ) -> List[dict]:
         logger.info(
             f"Extracting variables metadata from Define-XML. domain_name={domain_name}"
         )
         try:
             metadata = self._odm_loader.MetaDataVersion()
-            domain_metadata = self._get_domain_metadata(metadata, domain_name)
+            domain_metadata = self._get_domain_metadata(metadata, domain_name, name)
             variables_metadata = []
             codelist_map = self._get_codelist_def_map(metadata.CodeList)
             for index, itemref in enumerate(domain_metadata.ItemRef):
@@ -267,11 +269,18 @@ def _get_dataset_metadata(self, metadata, dataset_name):
                 f"Dataset {dataset_name} is not found in Define XML"
             )
 
-    def _get_domain_metadata(self, metadata, domain_name):
+    def _get_domain_metadata(self, metadata, domain_name, name: str = None):
         try:
-            domain_metadata = next(
-                item for item in metadata.ItemGroupDef if item.Domain == domain_name
-            )
+            if name:
+                domain_metadata = next(
+                    item
+                    for item in metadata.ItemGroupDef
+                    if item.Domain == domain_name and item.Name == name
+                )
+            else:
+                domain_metadata = next(
+                    item for item in metadata.ItemGroupDef if item.Domain == domain_name
+                )
             return domain_metadata
         except StopIteration:
             raise DomainNotFoundInDefineXMLError(

cdisc_rules_engine/utilities/decorators.py

@@ -48,13 +48,20 @@ def cached(cache_key: str):  # noqa: C901
     """
 
     def format_cache_key(
-        key: str, args=[], study_id=None, data_bundle_id=None, domain_name=None
+        key: str,
+        args=[],
+        study_id=None,
+        data_bundle_id=None,
+        domain_name=None,
+        name=None,
     ):
         """
         If a study_id and data_bundle_id are available,
         cache_key = {study_id}/{data_bundle_id}/key
         else the function just returns the provided cache key.
         """
+        if name:
+            key = f"{name}/" + key
         if domain_name:
             key = f"{domain_name}/" + key
         if data_bundle_id:
@@ -85,6 +92,7 @@ def inner(*args, **kwargs):
                 if hasattr(instance, "domain")
                 else kwargs.get("domain_name")
             )
+            name = instance.name if hasattr(instance, "name") else kwargs.get("name")
             if (
                 hasattr(instance, "cache_service")
                 and instance.cache_service is not None
@@ -95,6 +103,7 @@ def inner(*args, **kwargs):
                     study_id=study_id,
                     data_bundle_id=data_bundle_id,
                     domain_name=domain_name,
+                    name=name,
                 )
                 cached_data = instance.cache_service.get(key)
                 if cached_data is not None:

tests/QARegressionTests/test_Issues/test_CoreIssue1421.py

@@ -0,0 +1,131 @@
+import os
+import subprocess
+import openpyxl
+import pytest
+from conftest import get_python_executable
+from QARegressionTests.globals import (
+    dataset_details_sheet,
+    issue_datails_sheet,
+    rules_report_sheet,
+    issue_sheet_variable_column,
+    issue_sheet_coreid_column,
+)
+
+
+@pytest.mark.regression
+def test_validate_define_xml_against_lib_metadata():
+    command = [
+        f"{get_python_executable()}",
+        "-m",
+        "core",
+        "validate",
+        "-s",
+        "sdtmig",
+        "-v",
+        "3-4",
+        "-dp",
+        os.path.join(
+            "tests",
+            "resources",
+            "CoreIssue1421",
+            "Dataset.json",
+        ),
+        "-lr",
+        os.path.join("tests", "resources", "CoreIssue1421", "Rule.yml"),
+        "-dxp",
+        os.path.join("tests", "resources", "CoreIssue1421", "Define.xml"),
+    ]
+    subprocess.run(command, check=True)
+
+    # Get the latest created Excel file
+    files = os.listdir()
+    excel_files = [
+        file
+        for file in files
+        if file.startswith("CORE-Report-") and file.endswith(".xlsx")
+    ]
+    excel_file_path = sorted(excel_files)[-1]
+    # Open the Excel file
+    workbook = openpyxl.load_workbook(excel_file_path)
+
+    # Go to the "Issue Details" sheet
+    sheet = workbook[issue_datails_sheet]
+
+    variables_values_column = sheet[issue_sheet_variable_column]
+    variables_values = [
+        cell.value for cell in variables_values_column[1:] if cell.value is not None
+    ]
+    assert len(variables_values) == 1
+    for value in variables_values:
+        assert len(value.split(",")) == 6
+
+    variables_names_column = sheet["H"]
+    variables_names_values = [
+        cell.value for cell in variables_names_column[1:] if cell.value is not None
+    ]
+    assert len(variables_names_values) == 1
+    for value in variables_names_values:
+        assert len(value.split(",")) == 6
+
+    dataset_column = sheet["D"]
+    dataset_column_values = [
+        cell.value for cell in dataset_column[1:] if cell.value is not None
+    ]
+    assert sorted(set(dataset_column_values)) == ["dm.xpt"]
+
+    core_id_column = sheet[issue_sheet_coreid_column]
+    core_id_column_values = [
+        cell.value for cell in core_id_column[1:] if cell.value is not None
+    ]
+    assert set(core_id_column_values) == {"CDISC.SDTMIG.CG0999"}
+
+    # Go to the "Rules Report" sheet
+    rules_values = [
+        row for row in workbook[rules_report_sheet].iter_rows(values_only=True)
+    ][1:]
+    rules_values = [row for row in rules_values if any(row)]
+    assert rules_values[0][0] == "CDISC.SDTMIG.CG0999"
+    assert "SUCCESS" in rules_values[0]
+    assert (
+        rules_values[0][4]
+        == "Issue with codelist definition in the Define-XML document."
+    )
+
+    # Go to the "Dataset Details" sheet
+    dataset_sheet = workbook[dataset_details_sheet]
+    dataset_values = [row for row in dataset_sheet.iter_rows(values_only=True)][1:]
+    dataset_values = [row for row in dataset_values if any(row)]
+    assert len(dataset_values) > 0
+    dataset_names = set(row[0] for row in dataset_values if row[0] is not None)
+    assert dataset_names == {"ae.xpt", "dm.xpt", "ec.xpt", "ex.xpt", "suppec.xpt"}
+    expected_records = {
+        "ae.xpt": 74,
+        "dm.xpt": 18,
+        "ec.xpt": 1590,
+        "ex.xpt": 1583,
+        "suppec.xpt": 13,
+    }
+    for row in dataset_values:
+        dataset_name = row[0]
+        records_count = row[-1]
+        assert records_count == expected_records[dataset_name]
+
+    # Go to the "Issue Summary" sheet
+    issue_summary_sheet = workbook["Issue Summary"]
+    summary_values = [row for row in issue_summary_sheet.iter_rows(values_only=True)][
+        1:
+    ]
+    summary_values = [row for row in summary_values if any(row)]
+    assert len(summary_values) == 1
+    core_ids = set(row[1] for row in summary_values if row[1] is not None)
+    assert core_ids == {"CDISC.SDTMIG.CG0999"}
+    # Check Message and dataset columns
+    assert (
+        summary_values[0][2]
+        == "Issue with codelist definition in the Define-XML document."
+    )
+    assert summary_values[0][0] == "dm.xpt"
+
+    # Delete the excel file
+    if os.path.exists(excel_file_path):
+        os.remove(excel_file_path)