Fix(dbt_cli): Make multiple --exclude's actually work

Author: erindru

sqlmesh_dbt/cli.py

@@ -46,10 +46,19 @@ def _cleanup() -> None:
 @click.group(invoke_without_command=True)
 @click.option("--profile", help="Which existing profile to load. Overrides output.profile")
 @click.option("-t", "--target", help="Which target to load for the given profile")
+@click.option(
+    "-d",
+    "--debug/--no-debug",
+    default=False,
+    help="Display debug logging during dbt execution. Useful for debugging and making bug reports events to help when debugging.",
+)
 @click.pass_context
 @cli_global_error_handler
 def dbt(
-    ctx: click.Context, profile: t.Optional[str] = None, target: t.Optional[str] = None
+    ctx: click.Context,
+    profile: t.Optional[str] = None,
+    target: t.Optional[str] = None,
+    debug: bool = False,
 ) -> None:
     """
     An ELT tool for managing your SQL transformations and data models, powered by the SQLMesh engine.
@@ -61,7 +70,9 @@ def dbt(
 
     # we have a partially applied function here because subcommands might set extra options like --vars
     # that need to be known before we attempt to load the project
-    ctx.obj = functools.partial(create, project_dir=Path.cwd(), profile=profile, target=target)
+    ctx.obj = functools.partial(
+        create, project_dir=Path.cwd(), profile=profile, target=target, debug=debug
+    )
 
     if not ctx.invoked_subcommand:
         if profile or target:

sqlmesh_dbt/operations.py

@@ -16,9 +16,10 @@
 
 
 class DbtOperations:
-    def __init__(self, sqlmesh_context: Context, dbt_project: Project):
+    def __init__(self, sqlmesh_context: Context, dbt_project: Project, debug: bool = False):
         self.context = sqlmesh_context
         self.project = dbt_project
+        self.debug = debug
 
     def list_(
         self,
@@ -55,6 +56,10 @@ def _selected_models(
         self, select: t.Optional[t.List[str]] = None, exclude: t.Optional[t.List[str]] = None
     ) -> t.Dict[str, Model]:
         if sqlmesh_selector := selectors.to_sqlmesh(select or [], exclude or []):
+            if self.debug:
+                self.console.print(f"dbt --select: {select}")
+                self.console.print(f"dbt --exclude: {exclude}")
+                self.console.print(f"sqlmesh equivalent: '{sqlmesh_selector}'")
             model_selector = self.context._new_selector()
             selected_models = {
                 fqn: model
@@ -119,7 +124,7 @@ def create(
         # so that DbtOperations can query information from the DBT project files in order to invoke SQLMesh correctly
         dbt_project = dbt_loader._projects[0]
 
-        return DbtOperations(sqlmesh_context, dbt_project)
+        return DbtOperations(sqlmesh_context, dbt_project, debug=debug)
 
 
 def init_project_if_required(project_dir: Path) -> None:

sqlmesh_dbt/selectors.py

@@ -37,15 +37,24 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) ->
             -> "+main.model_a & ^(raw.src_data)"
         --select "+main.model_a" --select "main.*b+" --exclude "raw.src_data"
             -> "(+main.model_a | main.*b+) & ^(raw.src_data)"
+        --select "+main.model_a" --select "main.*b+" --exclude "raw.src_data" --exclude "main.model_c"
+            -> "(+main.model_a | main.*b+) & ^(raw.src_data | main.model_c)"
+        --select "+main.model_a main.*b+" --exclude "raw.src_data main.model_c"
+            -> "(+main.model_a | main.*b+) & ^(raw.src_data | main.model_c)"
     """
     if not dbt_select and not dbt_exclude:
         return None
 
     select_expr = " | ".join(_to_sqlmesh(expr) for expr in dbt_select)
     select_expr = _wrap(select_expr) if dbt_exclude and len(dbt_select) > 1 else select_expr
 
-    exclude_expr = " | ".join(_to_sqlmesh(expr, negate=True) for expr in dbt_exclude)
-    exclude_expr = _wrap(exclude_expr) if dbt_select and len(dbt_exclude) > 1 else exclude_expr
+    exclude_expr = ""
+
+    if dbt_exclude:
+        exclude_expr = " | ".join(_to_sqlmesh(expr) for expr in dbt_exclude)
+        exclude_expr = _negate(
+            _wrap(exclude_expr) if dbt_select and len(dbt_exclude) > 1 else exclude_expr
+        )
 
     main_expr = " & ".join([expr for expr in [select_expr, exclude_expr] if expr])
 
@@ -56,13 +65,9 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) ->
     return main_expr
 
 
-def _to_sqlmesh(selector_str: str, negate: bool = False) -> str:
+def _to_sqlmesh(selector_str: str) -> str:
     unions, intersections = _split_unions_and_intersections(selector_str)
 
-    if negate:
-        unions = [_negate(u) for u in unions]
-        intersections = [_negate(i) for i in intersections]
-
     union_expr = " | ".join(unions)
     intersection_expr = " & ".join(intersections)
 
@@ -79,6 +84,7 @@ def _split_unions_and_intersections(selector_str: str) -> t.Tuple[t.List[str], t
     # break space-separated items like: "my_first_model my_second_model" into a list of selectors to union
     # and comma-separated items like: "my_first_model,my_second_model" into a list of selectors to intersect
     # but, take into account brackets, eg "(my_first_model & my_second_model)" should not be split
+    # also take into account both types in the same string, eg "my_first_model my_second_model model_3,model_4,model_5"
 
     def _split_by(input: str, delimiter: str) -> t.Iterator[str]:
         buf = ""

tests/dbt/cli/test_list.py

@@ -34,6 +34,7 @@ def test_list_select(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Resul
 
 
 def test_list_select_exclude(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Result]):
+    # single exclude
     result = invoke_cli(["list", "--select", "main.raw_customers+", "--exclude", "main.orders"])
 
     assert result.exit_code == 0
@@ -47,6 +48,20 @@ def test_list_select_exclude(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..
     assert "main.stg_payments" not in result.output
     assert "main.raw_orders" not in result.output
 
+    # multiple exclude
+    for args in (
+        ["--select", "main.stg_orders+", "--exclude", "main.customers", "--exclude", "main.orders"],
+        ["--select", "main.stg_orders+", "--exclude", "main.customers main.orders"],
+    ):
+        result = invoke_cli(["list", *args])
+        assert result.exit_code == 0
+        assert not result.exception
+
+        assert "main.stg_orders" in result.output
+
+        assert "main.customers" not in result.output
+        assert "main.orders" not in result.output
+
 
 def test_list_with_vars(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Result]):
     (jaffle_shop_duckdb / "models" / "aliased_model.sql").write_text("""

tests/dbt/cli/test_selectors.py

@@ -27,10 +27,11 @@ def test_selection(dbt_select: t.List[str], expected: t.Optional[str]):
         ([], None),
         (["main.model_a"], "^(main.model_a)"),
         (["(main.model_a & main.model_b)"], "^(main.model_a & main.model_b)"),
-        (["main.model_a +main.model_b"], "^(main.model_a) | ^(+main.model_b)"),
+        (["main.model_a,main.model_b"], "^(main.model_a & main.model_b)"),
+        (["main.model_a +main.model_b"], "^(main.model_a | +main.model_b)"),
         (
             ["(+main.model_a & ^main.model_b)", "main.model_c"],
-            "^(+main.model_a & ^main.model_b) | ^(main.model_c)",
+            "^((+main.model_a & ^main.model_b) | main.model_c)",
         ),
     ],
 )
@@ -51,7 +52,7 @@ def test_exclusion(dbt_exclude: t.List[str], expected: t.Optional[str]):
         (
             ["+main.model_a", "main.*b+"],
             ["raw.src_data", "tag:disabled"],
-            "(+main.model_a | main.*b+) & (^(raw.src_data) | ^(tag:disabled))",
+            "(+main.model_a | main.*b+) & ^(raw.src_data | tag:disabled)",
         ),
     ],
 )