feat: dbt adapter allow invalid source ref for tests (#5259)

Author: eakmanrq

sqlmesh/dbt/basemodel.py

@@ -247,7 +247,7 @@ def tests_ref_source_dependencies(self) -> Dependencies:
 
     def remove_tests_with_invalid_refs(self, context: DbtContext) -> None:
         """
-        Removes tests that reference models that do not exist in the context in order to match dbt behavior.
+        Removes tests that reference models or sources that do not exist in the context in order to match dbt behavior.
 
         Args:
             context: The dbt context this model resides within.
@@ -259,6 +259,7 @@ def remove_tests_with_invalid_refs(self, context: DbtContext) -> None:
             test
             for test in self.tests
             if all(ref in context.refs for ref in test.dependencies.refs)
+            and all(source in context.sources for source in test.dependencies.sources)
         ]
 
     def check_for_circular_test_refs(self, context: DbtContext) -> None:

sqlmesh/dbt/context.py

@@ -12,7 +12,7 @@
 from sqlmesh.dbt.manifest import ManifestHelper
 from sqlmesh.dbt.target import TargetConfig
 from sqlmesh.utils import AttributeDict
-from sqlmesh.utils.errors import ConfigError, SQLMeshError, MissingModelError
+from sqlmesh.utils.errors import ConfigError, SQLMeshError, MissingModelError, MissingSourceError
 from sqlmesh.utils.jinja import (
     JinjaGlobalAttribute,
     JinjaMacroRegistry,
@@ -266,14 +266,13 @@ def context_for_dependencies(self, dependencies: Dependencies) -> DbtContext:
                 else:
                     models[ref] = t.cast(ModelConfig, model)
             else:
-                exception = MissingModelError(ref)
-                raise exception
+                raise MissingModelError(ref)
 
         for source in dependencies.sources:
             if source in self.sources:
                 sources[source] = self.sources[source]
             else:
-                raise ConfigError(f"Source '{source}' was not found.")
+                raise MissingSourceError(source)
 
         variables = {k: v for k, v in self.variables.items() if k in dependencies.variables}
 

sqlmesh/dbt/loader.py

@@ -23,7 +23,7 @@
 from sqlmesh.dbt.project import Project
 from sqlmesh.dbt.target import TargetConfig
 from sqlmesh.utils import UniqueKeyDict
-from sqlmesh.utils.errors import ConfigError, MissingModelError
+from sqlmesh.utils.errors import ConfigError, MissingModelError, BaseMissingReferenceError
 from sqlmesh.utils.jinja import (
     JinjaMacroRegistry,
     make_jinja_registry,
@@ -161,11 +161,13 @@ def _load_audits(
                     logger.debug("Converting '%s' to sqlmesh format", test.name)
                     try:
                         audits[test.name] = test.to_sqlmesh(package_context)
-                    except MissingModelError as e:
+                    except BaseMissingReferenceError as e:
+                        ref_type = "model" if isinstance(e, MissingModelError) else "source"
                         logger.warning(
-                            "Skipping audit '%s' because model '%s' is not a valid ref",
+                            "Skipping audit '%s' because %s '%s' is not a valid ref",
                             test.name,
-                            e.model_name,
+                            ref_type,
+                            e.ref,
                         )
 
         return audits

sqlmesh/utils/errors.py

@@ -33,12 +33,17 @@ def __init__(self, message: str | Exception, location: t.Optional[Path] = None)
             self.location = Path(location) if isinstance(location, str) else location
 
 
-class MissingModelError(ConfigError):
+class BaseMissingReferenceError(ConfigError):
+    def __init__(self, ref: str) -> None:
+        self.ref = ref
+
+
+class MissingModelError(BaseMissingReferenceError):
     """Raised when a model that is referenced is missing."""
 
-    def __init__(self, model_name: str) -> None:
-        self.model_name = model_name
-        super().__init__(f"Model '{model_name}' was not found.")
+
+class MissingSourceError(BaseMissingReferenceError):
+    """Raised when a source that is referenced is missing."""
 
 
 class MissingDependencyError(SQLMeshError):

tests/dbt/test_model.py

@@ -87,8 +87,15 @@ def test_load_invalid_ref_audit_constraints(
                                 "relationships": {
                                     "to": "ref('not_real_model')",
                                     "field": "cola",
-                                }
-                            }
+                                },
+                            },
+                            {
+                                # Reference a source that doesn't exist
+                                "relationships": {
+                                    "to": "source('not_real_source', 'not_real_table')",
+                                    "field": "cola",
+                                },
+                            },
                         ],
                     }
                 ],
@@ -134,6 +141,10 @@ def test_load_invalid_ref_audit_constraints(
         "Skipping audit 'relationships_full_model_cola__cola__ref_not_real_model_' because model 'not_real_model' is not a valid ref"
         in caplog.text
     )
+    assert (
+        "Skipping audit 'relationships_full_model_cola__cola__source_not_real_source_not_real_table_' because source 'not_real_source.not_real_table' is not a valid ref"
+        in caplog.text
+    )
     fqn = '"local"."main"."full_model"'
     assert fqn in context.snapshots
     # The audit isn't loaded due to the invalid ref