diff --git a/sqlmesh_dbt/cli.py b/sqlmesh_dbt/cli.py index d82c2afd92..2daa3f9d54 100644 --- a/sqlmesh_dbt/cli.py +++ b/sqlmesh_dbt/cli.py @@ -46,10 +46,19 @@ def _cleanup() -> None: @click.group(invoke_without_command=True) @click.option("--profile", help="Which existing profile to load. Overrides output.profile") @click.option("-t", "--target", help="Which target to load for the given profile") +@click.option( + "-d", + "--debug/--no-debug", + default=False, + help="Display debug logging during dbt execution. Useful for debugging and making bug reports events to help when debugging.", +) @click.pass_context @cli_global_error_handler def dbt( - ctx: click.Context, profile: t.Optional[str] = None, target: t.Optional[str] = None + ctx: click.Context, + profile: t.Optional[str] = None, + target: t.Optional[str] = None, + debug: bool = False, ) -> None: """ An ELT tool for managing your SQL transformations and data models, powered by the SQLMesh engine. @@ -61,7 +70,9 @@ def dbt( # we have a partially applied function here because subcommands might set extra options like --vars # that need to be known before we attempt to load the project - ctx.obj = functools.partial(create, project_dir=Path.cwd(), profile=profile, target=target) + ctx.obj = functools.partial( + create, project_dir=Path.cwd(), profile=profile, target=target, debug=debug + ) if not ctx.invoked_subcommand: if profile or target: diff --git a/sqlmesh_dbt/operations.py b/sqlmesh_dbt/operations.py index 296000847c..270bba6511 100644 --- a/sqlmesh_dbt/operations.py +++ b/sqlmesh_dbt/operations.py @@ -16,9 +16,10 @@ class DbtOperations: - def __init__(self, sqlmesh_context: Context, dbt_project: Project): + def __init__(self, sqlmesh_context: Context, dbt_project: Project, debug: bool = False): self.context = sqlmesh_context self.project = dbt_project + self.debug = debug def list_( self, @@ -55,6 +56,10 @@ def _selected_models( self, select: t.Optional[t.List[str]] = None, exclude: t.Optional[t.List[str]] = None ) -> t.Dict[str, Model]: if sqlmesh_selector := selectors.to_sqlmesh(select or [], exclude or []): + if self.debug: + self.console.print(f"dbt --select: {select}") + self.console.print(f"dbt --exclude: {exclude}") + self.console.print(f"sqlmesh equivalent: '{sqlmesh_selector}'") model_selector = self.context._new_selector() selected_models = { fqn: model @@ -119,7 +124,7 @@ def create( # so that DbtOperations can query information from the DBT project files in order to invoke SQLMesh correctly dbt_project = dbt_loader._projects[0] - return DbtOperations(sqlmesh_context, dbt_project) + return DbtOperations(sqlmesh_context, dbt_project, debug=debug) def init_project_if_required(project_dir: Path) -> None: diff --git a/sqlmesh_dbt/selectors.py b/sqlmesh_dbt/selectors.py index 16f5c2ea98..120d5dcb36 100644 --- a/sqlmesh_dbt/selectors.py +++ b/sqlmesh_dbt/selectors.py @@ -37,6 +37,10 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) -> -> "+main.model_a & ^(raw.src_data)" --select "+main.model_a" --select "main.*b+" --exclude "raw.src_data" -> "(+main.model_a | main.*b+) & ^(raw.src_data)" + --select "+main.model_a" --select "main.*b+" --exclude "raw.src_data" --exclude "main.model_c" + -> "(+main.model_a | main.*b+) & ^(raw.src_data | main.model_c)" + --select "+main.model_a main.*b+" --exclude "raw.src_data main.model_c" + -> "(+main.model_a | main.*b+) & ^(raw.src_data | main.model_c)" """ if not dbt_select and not dbt_exclude: return None @@ -44,8 +48,13 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) -> select_expr = " | ".join(_to_sqlmesh(expr) for expr in dbt_select) select_expr = _wrap(select_expr) if dbt_exclude and len(dbt_select) > 1 else select_expr - exclude_expr = " | ".join(_to_sqlmesh(expr, negate=True) for expr in dbt_exclude) - exclude_expr = _wrap(exclude_expr) if dbt_select and len(dbt_exclude) > 1 else exclude_expr + exclude_expr = "" + + if dbt_exclude: + exclude_expr = " | ".join(_to_sqlmesh(expr) for expr in dbt_exclude) + exclude_expr = _negate( + _wrap(exclude_expr) if dbt_select and len(dbt_exclude) > 1 else exclude_expr + ) main_expr = " & ".join([expr for expr in [select_expr, exclude_expr] if expr]) @@ -56,13 +65,9 @@ def to_sqlmesh(dbt_select: t.Collection[str], dbt_exclude: t.Collection[str]) -> return main_expr -def _to_sqlmesh(selector_str: str, negate: bool = False) -> str: +def _to_sqlmesh(selector_str: str) -> str: unions, intersections = _split_unions_and_intersections(selector_str) - if negate: - unions = [_negate(u) for u in unions] - intersections = [_negate(i) for i in intersections] - union_expr = " | ".join(unions) intersection_expr = " & ".join(intersections) @@ -79,6 +84,7 @@ def _split_unions_and_intersections(selector_str: str) -> t.Tuple[t.List[str], t # break space-separated items like: "my_first_model my_second_model" into a list of selectors to union # and comma-separated items like: "my_first_model,my_second_model" into a list of selectors to intersect # but, take into account brackets, eg "(my_first_model & my_second_model)" should not be split + # also take into account both types in the same string, eg "my_first_model my_second_model model_3,model_4,model_5" def _split_by(input: str, delimiter: str) -> t.Iterator[str]: buf = "" diff --git a/tests/dbt/cli/test_list.py b/tests/dbt/cli/test_list.py index e854954903..3701822b4c 100644 --- a/tests/dbt/cli/test_list.py +++ b/tests/dbt/cli/test_list.py @@ -34,6 +34,7 @@ def test_list_select(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Resul def test_list_select_exclude(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Result]): + # single exclude result = invoke_cli(["list", "--select", "main.raw_customers+", "--exclude", "main.orders"]) assert result.exit_code == 0 @@ -47,6 +48,20 @@ def test_list_select_exclude(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[.. assert "main.stg_payments" not in result.output assert "main.raw_orders" not in result.output + # multiple exclude + for args in ( + ["--select", "main.stg_orders+", "--exclude", "main.customers", "--exclude", "main.orders"], + ["--select", "main.stg_orders+", "--exclude", "main.customers main.orders"], + ): + result = invoke_cli(["list", *args]) + assert result.exit_code == 0 + assert not result.exception + + assert "main.stg_orders" in result.output + + assert "main.customers" not in result.output + assert "main.orders" not in result.output + def test_list_with_vars(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Result]): (jaffle_shop_duckdb / "models" / "aliased_model.sql").write_text(""" diff --git a/tests/dbt/cli/test_selectors.py b/tests/dbt/cli/test_selectors.py index e494ed98a3..6041a50d0a 100644 --- a/tests/dbt/cli/test_selectors.py +++ b/tests/dbt/cli/test_selectors.py @@ -27,10 +27,11 @@ def test_selection(dbt_select: t.List[str], expected: t.Optional[str]): ([], None), (["main.model_a"], "^(main.model_a)"), (["(main.model_a & main.model_b)"], "^(main.model_a & main.model_b)"), - (["main.model_a +main.model_b"], "^(main.model_a) | ^(+main.model_b)"), + (["main.model_a,main.model_b"], "^(main.model_a & main.model_b)"), + (["main.model_a +main.model_b"], "^(main.model_a | +main.model_b)"), ( ["(+main.model_a & ^main.model_b)", "main.model_c"], - "^(+main.model_a & ^main.model_b) | ^(main.model_c)", + "^((+main.model_a & ^main.model_b) | main.model_c)", ), ], ) @@ -51,7 +52,7 @@ def test_exclusion(dbt_exclude: t.List[str], expected: t.Optional[str]): ( ["+main.model_a", "main.*b+"], ["raw.src_data", "tag:disabled"], - "(+main.model_a | main.*b+) & (^(raw.src_data) | ^(tag:disabled))", + "(+main.model_a | main.*b+) & ^(raw.src_data | tag:disabled)", ), ], )