Custom Domain Logic for Library

cdisc_rules_engine/dataset_builders/base_dataset_builder.py

@@ -204,6 +204,11 @@ def get_library_variables_metadata(self) -> DatasetInterface:
         variables: List[dict] = sdtm_utilities.get_variables_metadata_from_standard(
             domain=self.dataset_metadata.unsplit_name,
             library_metadata=self.library_metadata,
+            data_service=self.data_service,
+            dataset=self.get_dataset_contents(),
+            datasets=self.datasets,
+            dataset_metadata=self.dataset_metadata,
+            dataset_path=self.dataset_path,
         )
         variables_metadata: dict = self.library_metadata.variables_metadata.get(
             domain, {}

cdisc_rules_engine/operations/base_operation.py

@@ -228,8 +228,15 @@ def _get_variables_metadata_from_standard(self) -> List[dict]:
         # self.params.domain is unsplit_name
         domain_for_library = self.params.domain
         return sdtm_utilities.get_variables_metadata_from_standard(
-            domain_for_library,
-            self.library_metadata,
+            domain=domain_for_library,
+            library_metadata=self.library_metadata,
+            data_service=self.data_service,
+            dataset=self.evaluation_dataset,
+            dataset_metadata=self.data_service.get_raw_dataset_metadata(
+                dataset_name=self.params.dataset_path, datasets=self.params.datasets
+            ),
+            datasets=self.params.datasets,
+            dataset_path=self.params.dataset_path,
         )
 
     def get_allowed_variable_permissibility(self, variable_metadata: dict):
@@ -286,6 +293,9 @@ def _get_variables_metadata_from_standard_model(
             dataset_path=self.params.dataset_path,
             data_service=self.data_service,
             library_metadata=self.library_metadata,
+            dataset_metadata=self.data_service.get_raw_dataset_metadata(
+                dataset_name=self.params.dataset_path, datasets=self.params.datasets
+            ),
         )
 
     @staticmethod

cdisc_rules_engine/services/data_services/base_data_service.py

@@ -183,7 +183,7 @@ def get_dataset_class(
             name = class_data.get("name")
             if name:
                 return convert_library_class_name_to_ct_class(name)
-        return self._handle_special_cases(
+        return self._handle_custom_domains(
             dataset, dataset_metadata, file_path, datasets
         )
 
@@ -229,7 +229,7 @@ def get_dataset_metadata(
         }
         return self.dataset_implementation.from_dict(metadata_to_return)
 
-    def _handle_special_cases(
+    def _handle_custom_domains(
         self,
         dataset: DatasetInterface,
         dataset_metadata: SDTMDatasetMetadata,

cdisc_rules_engine/utilities/sdtm_utilities.py

@@ -16,6 +16,7 @@
 )
 import copy
 from cdisc_rules_engine.models.sdtm_dataset_metadata import SDTMDatasetMetadata
+from cdisc_rules_engine.models.dataset.dataset_interface import DatasetInterface
 from typing import Iterable, Tuple, List, Optional
 
 
@@ -53,7 +54,15 @@ def get_tabulation_model_type_and_version(model_link: dict) -> Tuple:
     return model_type, model_version
 
 
-def get_variables_metadata_from_standard(domain, library_metadata):  # noqa
+def get_variables_metadata_from_standard(  # noqa
+    domain,
+    library_metadata,
+    data_service,
+    dataset: DatasetInterface,
+    dataset_metadata: SDTMDatasetMetadata,
+    dataset_path: str,
+    datasets: Iterable[SDTMDatasetMetadata],
+):
     add_AP = False
     original_domain = domain
     if (
@@ -70,15 +79,24 @@ def get_variables_metadata_from_standard(domain, library_metadata):  # noqa
         domain = domain[2:]
         original_domain = domain
         add_AP = True
-
     standard_details = library_metadata.standard_metadata
     model_details = library_metadata.model_metadata
     is_custom = domain not in standard_details.get("domains", {})
     variables_metadata = []
-    IG_class_details, IG_domain_details = get_class_and_domain_metadata(
-        standard_details, domain
-    )
-    class_name = convert_library_class_name_to_ct_class(IG_class_details.get("name"))
+    if not is_custom:
+        IG_class_details, IG_domain_details = get_class_and_domain_metadata(
+            standard_details, domain
+        )
+        class_name = convert_library_class_name_to_ct_class(
+            IG_class_details.get("name")
+        )
+    else:
+        class_name = data_service._handle_custom_domains(
+            data_service.get_dataset(dataset_name=dataset_metadata.full_path),
+            dataset_metadata,
+            dataset_path,
+            datasets,
+        )
     model_class_details = get_class_metadata(model_details, class_name)
     # Both custom and standard General Observations pull from model
     if is_custom or class_name in DETECTABLE_CLASSES:
@@ -282,11 +300,13 @@ def get_variables_metadata_from_standard_model(  # noqa
     dataset_path: str,
     data_service: DataServiceInterface,
     library_metadata: LibraryMetadataContainer,
+    dataset_metadata: SDTMDatasetMetadata,
 ) -> List[dict]:
     """
     gets class via the IG then uses the class to get the variables via the model
     classes outside of general observation, we check the model for their definition
     if they are not there, differ to the standard definition of the domain
+    if custom, IDs class and uses class variables.
     """
     add_AP = False
     original_domain = domain
@@ -306,11 +326,18 @@ def get_variables_metadata_from_standard_model(  # noqa
         add_AP = True
     standard_details = library_metadata.standard_metadata
     model_details = library_metadata.model_metadata
-
-    IG_class_details, IG_domain_details = get_class_and_domain_metadata(
-        standard_details, domain
-    )
-    class_name = convert_library_class_name_to_ct_class(IG_class_details.get("name"))
+    is_custom = domain not in standard_details.get("domains", {})
+    if not is_custom:
+        IG_class_details, IG_domain_details = get_class_and_domain_metadata(
+            standard_details, domain
+        )
+        class_name = convert_library_class_name_to_ct_class(
+            IG_class_details.get("name")
+        )
+    else:
+        class_name = data_service._handle_custom_domains(
+            dataframe, dataset_metadata, dataset_path, datasets
+        )
     if class_name in DETECTABLE_CLASSES:
         model_class_details = get_class_metadata(model_details, class_name)
         (

tests/unit/test_operations/test_get_dataset_filtered_variables.py

@@ -96,6 +96,7 @@
     },
     {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
@@ -130,7 +131,7 @@
         "VISITNUM": [1, 2, 1],
         "VISIT": ["Day 1", "Day 7", "Day 1"],
     },
-    {"name": "AE"},
+    {"name": "AE", "first_record": {"DOMAIN": "AE"}},
     "role",
     "Timing",
     ["VISITNUM", "VISIT"],
@@ -188,6 +189,7 @@
     },
     {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
@@ -210,7 +212,7 @@
         "USUBJID": ["SUBJ001", "SUBJ002", "SUBJ003"],
         "AETERM": ["Headache", "Nausea", "Fatigue"],
     },
-    {"name": "AE"},
+    {"name": "AE", "first_record": {"DOMAIN": "AE"}},
     "role",
     "Identifier",
     ["STUDYID", "DOMAIN", "USUBJID", "AETERM"],
@@ -273,6 +275,7 @@
     },
     {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
@@ -298,7 +301,7 @@
         "AETERM": ["Headache", "Nausea"],
         "AESEQ": [1, 2],
     },
-    {"name": "AE"},
+    {"name": "AE", "first_record": {"DOMAIN": "AE"}},
     "role",
     "Identifier",
     ["STUDYID", "AESEQ"],
@@ -349,6 +352,7 @@
     },
     {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
@@ -370,7 +374,7 @@
         "AETERM": ["Headache", "Nausea"],
         "AESEQ": [1, 2],
     },
-    {"name": "AE"},
+    {"name": "AE", "first_record": {"DOMAIN": "AE"}},
     "role",
     "Timing",
     [],
@@ -438,6 +442,7 @@
     },
     {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"FA"},
         "classes": [
             {
                 "name": FINDINGS_ABOUT,
@@ -465,10 +470,10 @@
         "FASEQ": [1, 2],
         "USUBJID": ["SUBJ001", "SUBJ002"],
     },
-    {"name": "FA"},
+    {"name": "FA", "first_record": {"DOMAIN": "FA"}},
     "role",
     "Identifier",
-    ["STUDYID", "DOMAIN", "USUBJID"],
+    ["STUDYID", "DOMAIN", "FASEQ", "USUBJID"],
 )
 
 
@@ -638,6 +643,7 @@ def test_get_dataset_filtered_variables_dask(
 
     standard_metadata = {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
@@ -752,14 +758,19 @@ def test_get_dataset_filtered_variables_empty_dataset(
 
     standard_metadata = {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
                 "datasets": [
                     {
                         "name": "AE",
                         "datasetVariables": [
-                            {"name": "VISITNUM", "role": VariableRoles.TIMING.value},
+                            {
+                                "name": "VISITNUM",
+                                "role": VariableRoles.TIMING.value,
+                                "ordinal": 1,
+                            },
                         ],
                     }
                 ],
@@ -859,14 +870,19 @@ def test_get_dataset_filtered_variables_invalid_key(operation_params: OperationP
 
     standard_metadata = {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
                 "datasets": [
                     {
                         "name": "AE",
                         "datasetVariables": [
-                            {"name": "AETERM", "role": VariableRoles.IDENTIFIER.value},
+                            {
+                                "name": "AETERM",
+                                "role": VariableRoles.IDENTIFIER.value,
+                                "ordinal": 1,
+                            },
                         ],
                     }
                 ],

tests/unit/test_operations/test_get_model_filtered_variables.py

@@ -108,6 +108,7 @@
     },
     {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",
@@ -235,6 +236,7 @@
     },
     {
         "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+        "domains": {"AE"},
         "classes": [
             {
                 "name": "Events",

tests/unit/test_operations/test_library_model_column_order.py

@@ -76,6 +76,7 @@
 
 standard_metadata = {
     "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+    "domains": {"AE"},
     "classes": [
         {
             "name": "Events",
@@ -117,16 +118,31 @@ def test_get_column_order_from_library(operation_params: OperationParams, datase
     operation_params.domain = "AE"
     operation_params.standard = "sdtmig"
     operation_params.standard_version = "3-4"
+    operation_params.datasets = [
+        SDTMDatasetMetadata(name="AE", first_record={"DOMAIN": "AE"})
+    ]
 
     # save model metadata to cache
     cache = InMemoryCacheService.get_instance()
     library_metadata = LibraryMetadataContainer(
-        standard_metadata=standard_metadata, model_metadata=model_metadata
+        standard_metadata=standard_metadata,  # USE updated version
+        model_metadata=model_metadata,
     )
     # execute operation
-    data_service = LocalDataService.get_instance(
-        cache_service=cache, config=ConfigService()
+    data_service = LocalDataService(
+        cache_service=cache,
+        config=ConfigService(),
+        reader_factory=DataReaderFactory(),
+        standard="sdtmig",
+        standard_version="3-4",
+        library_metadata=library_metadata,
     )
+
+    def mock_get_raw_metadata(*args, **kwargs):
+        return SDTMDatasetMetadata(name="AE", first_record={"DOMAIN": "AE"})
+
+    data_service.get_raw_dataset_metadata = mock_get_raw_metadata
+
     operation = LibraryModelColumnOrder(
         operation_params,
         operation_params.dataframe,
@@ -215,6 +231,7 @@ def test_get_column_order_from_library(operation_params: OperationParams, datase
             },
             {
                 "_links": {"model": {"href": "/mdr/sdtm/1-5"}},
+                "domains": {"AE"},
                 "classes": [
                     {
                         "name": FINDINGS_ABOUT,
@@ -260,7 +277,9 @@ def test_get_findings_class_column_order_from_library(
     operation_params.domain = "AE"
     operation_params.standard = "sdtmig"
     operation_params.standard_version = "3-4"
-    operation_params.datasets = [SDTMDatasetMetadata(name="AE")]
+    operation_params.datasets = [
+        SDTMDatasetMetadata(name="AE", first_record={"DOMAIN": "AE"})
+    ]
 
     # save model metadata to cache
     cache = InMemoryCacheService.get_instance()
@@ -276,6 +295,11 @@ def test_get_findings_class_column_order_from_library(
         standard_version="3-4",
         library_metadata=library_metadata,
     )
+
+    def mock_get_raw_metadata(*args, **kwargs):
+        return SDTMDatasetMetadata(name="AE", first_record={"DOMAIN": "AE"})
+
+    data_service.get_raw_dataset_metadata = mock_get_raw_metadata
     operation = LibraryModelColumnOrder(
         operation_params,
         operation_params.dataframe,