fixes

.flake8

@@ -1,8 +1,7 @@
 [flake8]
-max-line-length = 120
 max_complexity = 10
-ignore = E203, W503, E713
-
+ignore = E203, W503, E713, E501
+# E203, E501, and W503 are ignored as they conflict with Black 
 exclude = .github,
     .pytest_cache,
     cdisc_rules_engine/resources,

.github/workflows/lint-format.yml

@@ -53,7 +53,7 @@ jobs:
       - name: Run flake8
         run: |
           flake8 ${{needs.get_changed_files.outputs.py}} --count --select=E9,F63,F7,F82 --show-source --statistics
-          flake8 ${{needs.get_changed_files.outputs.py}} --ignore E203,W503 --count --statistics
+          flake8 ${{needs.get_changed_files.outputs.py}} --count --statistics
       - name: Run black
         run: |
           black --check ${{needs.get_changed_files.outputs.py}}

README.md

@@ -394,6 +394,35 @@ Update locally stored cache data (An api-key can be provided through the environ
 
 To obtain an api key, please follow the instructions found here: <https://wiki.cdisc.org/display/LIBSUPRT/Getting+Started%3A+Access+to+CDISC+Library+API+using+API+Key+Authentication>. Please note it can take up to an hour after sign up to have an api key issued
 
+The update-cache command options are:
+
+```
+  -c, --cache-path TEXT                   Relative path to cache. Optional. Only required if the cache has been
+                                          moved from its default location.
+  --apikey TEXT                           CDISC Library api key.
+                                          Can also be provided as an environment
+                                          variable CDISC_LIBRARY_API_KEY
+  -crd, --custom-rules-directory TEXT     Relative path to directory containing local
+                                          rules in yaml or JSON formats to be added
+                                          to the cache
+  -cr, --custom-rule TEXT                 Relative path to rule file in yaml or JSON
+                                          formats to be added to the cache.
+                                          Can be specified multiple times.
+  -rcr, --remove-custom-rules TEXT        Remove rules from the cache. Can be a single
+                                          rule ID, a comma-separated list of IDs,
+                                          or 'ALL' to remove all custom rules.
+  -ucr, --update-custom-rule TEXT         Relative path to rule file in yaml or JSON
+                                          formats. Rule will be updated in cache
+                                          with this file.
+  -cs, --custom-standard TEXT             Relative path to JSON file containing custom
+                                          standard details. Will update the standard
+                                          if it already exists.
+  -cse, --custom-standard-encoding TEXT   Encoding for custom standard details.
+  -rcs, --remove-custom-standard TEXT     Removes a custom standard and version from
+                                          the cache. Can be specified multiple times.
+  --help                                  Show this message and exit.
+```
+
 ##### Custom Standards and Rules
 
 ###### Custom Rules Management

cdisc_rules_engine/check_operators/dataframe_operators.py

@@ -1371,48 +1371,6 @@ def is_ordered_set(self, other_value):
     def is_not_ordered_set(self, other_value):
         return ~self.is_ordered_set(other_value)
 
-    @log_operator_execution
-    @type_operator(FIELD_DATAFRAME)
-    def non_conformant_value_data_type(self, other_value):
-        results = False
-        for vlm in self.value_level_metadata:
-            results |= self.value.apply(
-                lambda row: vlm["filter"](row) and not vlm["type_check"](row), axis=1
-            )
-        return self.value.convert_to_series(results.values)
-
-    @log_operator_execution
-    @type_operator(FIELD_DATAFRAME)
-    def non_conformant_value_length(self, other_value):
-        results = False
-        for vlm in self.value_level_metadata:
-            results |= self.value.apply(
-                lambda row: vlm["filter"](row) and not vlm["length_check"](row), axis=1
-            )
-        return self.value.convert_to_series(results)
-
-    @log_operator_execution
-    @type_operator(FIELD_DATAFRAME)
-    def conformant_value_data_type(self, other_value):
-        results = False
-        for vlm in self.value_level_metadata:
-            results |= self.value.apply(
-                lambda row: vlm["filter"](row) and vlm["type_check"](row),
-                axis=1,
-                meta=pd.Series([True, False], dtype=bool),
-            ).fillna(False)
-        return self.value.convert_to_series(results)
-
-    @log_operator_execution
-    @type_operator(FIELD_DATAFRAME)
-    def conformant_value_length(self, other_value):
-        results = False
-        for vlm in self.value_level_metadata:
-            results |= self.value.apply(
-                lambda row: vlm["filter"](row) and vlm["length_check"](row), axis=1
-            )
-        return self.value.convert_to_series(results)
-
     @log_operator_execution
     @type_operator(FIELD_DATAFRAME)
     def has_next_corresponding_record(self, other_value: dict):
@@ -1534,21 +1492,6 @@ def check_inconsistency(row):
 
         return df.apply(check_inconsistency, axis=1)
 
-    @log_operator_execution
-    @type_operator(FIELD_DATAFRAME)
-    def references_correct_codelist(self, other_value: dict):
-        target = other_value.get("target")
-        comparator = other_value.get("comparator")
-        result = self.value.apply(
-            lambda row: self.valid_codelist_reference(row[target], row[comparator]),
-            axis=1,
-        )
-        return result
-
-    @type_operator(FIELD_DATAFRAME)
-    def does_not_reference_correct_codelist(self, other_value: dict):
-        return ~self.references_correct_codelist(other_value)
-
     def next_column_exists_and_previous_is_null(self, row) -> bool:
         row.reset_index(drop=True, inplace=True)
         for index in row[

cdisc_rules_engine/dataset_builders/dataset_builder_factory.py

@@ -1,4 +1,3 @@
-# flake8: noqa
 from typing import Type
 
 from cdisc_rules_engine.dataset_builders.json_schema_check_dataset_builder import (
@@ -23,15 +22,9 @@
 from cdisc_rules_engine.dataset_builders.domain_list_dataset_builder import (
     DomainListDatasetBuilder,
 )
-from cdisc_rules_engine.dataset_builders.define_variables_dataset_builder import (
-    DefineVariablesDatasetBuilder,
-)
 from cdisc_rules_engine.dataset_builders.variables_metadata_with_define_dataset_builder import (
     VariablesMetadataWithDefineDatasetBuilder,
 )
-from cdisc_rules_engine.dataset_builders.define_item_group_dataset_builder import (
-    DefineItemGroupDatasetBuilder,
-)
 from cdisc_rules_engine.dataset_builders.contents_define_variables_dataset_builder import (
     ContentsDefineVariablesDatasetBuilder,
 )
@@ -65,17 +58,14 @@
 
 class DatasetBuilderFactory(FactoryInterface):
     _builders_map = {
+        RuleTypes.RECORD_CHECK.value: ContentsDatasetBuilder,
         RuleTypes.DATASET_CONTENTS_CHECK_AGAINST_DEFINE.value: ContentsDefineDatasetBuilder,
         RuleTypes.DATASET_METADATA_CHECK.value: ContentMetadataDatasetBuilder,
         RuleTypes.DATASET_METADATA_CHECK_AGAINST_DEFINE.value: DatasetMetadataDefineDatasetBuilder,
         RuleTypes.VARIABLE_METADATA_CHECK.value: VariablesMetadataDatasetBuilder,
         RuleTypes.DOMAIN_PRESENCE_CHECK.value: DomainListDatasetBuilder,
         RuleTypes.DOMAIN_PRESENCE_CHECK_AGAINST_DEFINE.value: DomainListWithDefineDatasetBuilder,
-        RuleTypes.DEFINE_ITEM_METADATA_CHECK.value: DefineVariablesDatasetBuilder,
         RuleTypes.VARIABLE_METADATA_CHECK_AGAINST_DEFINE.value: VariablesMetadataWithDefineDatasetBuilder,
-        RuleTypes.DATASET_CONTENTS_CHECK_AGAINST_DEFINE_AND_LIBRARY.value: ContentsDatasetBuilder,
-        RuleTypes.VALUE_LEVEL_METADATA_CHECK_AGAINST_DEFINE.value: ContentsDatasetBuilder,
-        RuleTypes.DEFINE_ITEM_GROUP_METADATA_CHECK.value: DefineItemGroupDatasetBuilder,
         RuleTypes.VALUE_CHECK_AGAINST_DEFINE_XML_VARIABLE.value: ContentsDefineVariablesDatasetBuilder,
         RuleTypes.VALUE_CHECK_AGAINST_DEFINE_XML_VLM.value: ContentsDefineVLMDatasetBuilder,
         RuleTypes.VARIABLE_METADATA_CHECK_AGAINST_LIBRARY.value: VariablesMetadataWithLibraryMetadataDatasetBuilder,
@@ -106,7 +96,11 @@ def get_service(
         """
         Get instance of dataset builder by name.
         """
-        builder = self._builders_map.get(name, ContentsDatasetBuilder)(
+        builder_class = self._builders_map.get(name)
+        if builder_class is None:
+            raise ValueError(f"Invalid Rule Type Entered: '{name}'")
+
+        return builder_class(
             kwargs.get("rule"),
             kwargs.get("data_service"),
             kwargs.get("cache_service"),
@@ -121,4 +115,3 @@ def get_service(
             kwargs.get("standard_substandard", None),
             kwargs.get("library_metadata"),
         )
-        return builder

cdisc_rules_engine/dataset_builders/variables_metadata_with_define_dataset_builder.py

@@ -29,6 +29,7 @@ def build(self):
         define_variable_codelist_coded_values,
         define_variable_codelist_coded_codes,
         define_variable_mandatory
+        define_variable_has_comment
         """
         # get Define XML metadata for domain and use it as a rule comparator
         variable_metadata: List[dict] = self.get_define_xml_variables_metadata()

cdisc_rules_engine/enums/rule_types.py

@@ -2,20 +2,13 @@
 
 
 class RuleTypes(BaseEnum):
-    DATASET_CONTENTS_CHECK_AGAINST_DEFINE_AND_LIBRARY = (
-        "Dataset Contents Check against Define XML and Library Metadata"
-    )
+    RECORD_CHECK = "Record Data"
     DATASET_CONTENTS_CHECK_AGAINST_DEFINE = "Dataset Contents Check against Define XML"
     DATASET_METADATA_CHECK = "Dataset Metadata Check"
     DATASET_METADATA_CHECK_AGAINST_DEFINE = "Dataset Metadata Check against Define XML"
-    DEFINE_ITEM_GROUP_METADATA_CHECK = "Define Item Group Metadata Check"
-    DEFINE_ITEM_METADATA_CHECK = "Define Item Metadata Check"
     DOMAIN_PRESENCE_CHECK = "Domain Presence Check"
     DOMAIN_PRESENCE_CHECK_AGAINST_DEFINE = "Domain Presence Check against Define XML"
     JSONATA = "JSONata"
-    VALUE_LEVEL_METADATA_CHECK_AGAINST_DEFINE = (
-        "Value Level Metadata Check against Define XML"
-    )
     VARIABLE_METADATA_CHECK = "Variable Metadata Check"
     VARIABLE_METADATA_CHECK_AGAINST_DEFINE = (
         "Variable Metadata Check against Define XML"

cdisc_rules_engine/rules_engine.py

@@ -344,54 +344,7 @@ def validate_rule(
             kwargs["codelist_term_maps"] = (
                 self.library_metadata.get_all_ct_package_metadata()
             )
-        if rule.get("rule_type") == RuleTypes.DEFINE_ITEM_METADATA_CHECK.value:
-            if self.library_metadata:
-                kwargs["variable_codelist_map"] = (
-                    self.library_metadata.variable_codelist_map
-                )
-                kwargs["codelist_term_maps"] = (
-                    self.library_metadata.get_all_ct_package_metadata()
-                )
-        elif (
-            rule.get("rule_type")
-            == RuleTypes.VARIABLE_METADATA_CHECK_AGAINST_DEFINE.value
-        ):
-            self.rule_processor.add_comparator_to_rule_conditions(
-                rule, comparator=None, target_prefix="define_"
-            )
-        elif (
-            rule.get("rule_type")
-            == RuleTypes.VALUE_LEVEL_METADATA_CHECK_AGAINST_DEFINE.value
-        ):
-            value_level_metadata: List[dict] = self.get_define_xml_value_level_metadata(
-                dataset_metadata.full_path, dataset_metadata.unsplit_name
-            )
-            kwargs["value_level_metadata"] = value_level_metadata
-
-        elif (
-            rule.get("rule_type")
-            == RuleTypes.DATASET_CONTENTS_CHECK_AGAINST_DEFINE_AND_LIBRARY.value
-        ):
-            library_metadata: dict = self.library_metadata.variables_metadata.get(
-                dataset_metadata.domain, {}
-            )
-            define_metadata: List[dict] = builder.get_define_xml_variables_metadata()
-            targets: List[str] = (
-                self.data_processor.filter_dataset_columns_by_metadata_and_rule(
-                    dataset.columns.tolist(), define_metadata, library_metadata, rule
-                )
-            )
-            rule_copy = deepcopy(rule)
-            updated_conditions = RuleProcessor.duplicate_conditions_for_all_targets(
-                rule_copy["conditions"], targets
-            )
-            rule_copy["conditions"].set_conditions(updated_conditions)
-            # When duplicating conditions,
-            # rule should be copied to prevent updates to concurrent rule executions
-            return self.execute_rule(
-                rule_copy, dataset, datasets, dataset_metadata, **kwargs
-            )
-        elif rule.get("rule_type") == RuleTypes.JSONATA.value:
+        if rule.get("rule_type") == RuleTypes.JSONATA.value:
             return JSONataProcessor.execute_jsonata_rule(
                 rule, dataset, self.jsonata_custom_functions
             )

cdisc_rules_engine/utilities/rule_processor.py

@@ -560,35 +560,6 @@ def add_operator_to_rule_conditions(
                 condition.clear()  # delete all keys from dict
                 condition[AllowedConditionsKeys.ANY.value] = nested_conditions
 
-    def add_comparator_to_rule_conditions(
-        self, rule: dict, comparator: dict = None, target_prefix=None
-    ):
-        """
-        Adds "comparator" key to rule conditions.value key.
-
-        comparator parameter is a dict where
-        keys are targets and values are comparators.
-
-        The rule is passed and changed by reference.
-        """
-        conditions: ConditionInterface = rule["conditions"]
-        for condition in conditions.values():
-            value: dict = condition["value"]
-            if comparator:
-                # Adding a specific value
-                comparator_to_add = comparator.get(value["target"])
-            elif target_prefix:
-                # Referencing a target variable in another dataset
-                comparator_to_add = f"{target_prefix}{value['target']}"
-            else:
-                comparator_to_add = None
-            if comparator_to_add:
-                value["comparator"] = comparator_to_add
-        logger.info(
-            f"Added comparator to rule conditions. "
-            f"comparator={comparator}, conditions={rule['conditions']}"
-        )
-
     def _preprocess_operation_params(
         self, operation_params: OperationParams, domain_details: dict = None
     ) -> OperationParams: