From deaad34577e4ad4946aeb2777657c2d63efaf092 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Mon, 11 Aug 2025 04:34:38 +0000 Subject: [PATCH 1/2] Feat: dbt cli --- .pre-commit-config.yaml | 2 +- pyproject.toml | 3 +- sqlmesh_dbt/__init__.py | 5 + sqlmesh_dbt/cli.py | 80 +++++++++++ sqlmesh_dbt/console.py | 8 ++ sqlmesh_dbt/operations.py | 133 ++++++++++++++++++ tests/dbt/cli/__init__.py | 1 + tests/dbt/cli/conftest.py | 29 ++++ .../jaffle_shop_duckdb/dbt_project.yml | 34 +++++ .../jaffle_shop_duckdb/models/customers.sql | 69 +++++++++ .../jaffle_shop_duckdb/models/docs.md | 14 ++ .../jaffle_shop_duckdb/models/orders.sql | 56 ++++++++ .../jaffle_shop_duckdb/models/overview.md | 11 ++ .../jaffle_shop_duckdb/models/schema.yml | 82 +++++++++++ .../models/staging/schema.yml | 31 ++++ .../models/staging/stg_customers.sql | 22 +++ .../models/staging/stg_orders.sql | 23 +++ .../models/staging/stg_payments.sql | 25 ++++ .../fixtures/jaffle_shop_duckdb/profiles.yml | 8 ++ .../jaffle_shop_duckdb/seeds/.gitkeep | 0 .../seeds/raw_customers.csv | 101 +++++++++++++ .../jaffle_shop_duckdb/seeds/raw_orders.csv | 100 +++++++++++++ .../jaffle_shop_duckdb/seeds/raw_payments.csv | 114 +++++++++++++++ tests/dbt/cli/test_list.py | 17 +++ tests/dbt/cli/test_operations.py | 57 ++++++++ tests/dbt/cli/test_run.py | 15 ++ 26 files changed, 1038 insertions(+), 2 deletions(-) create mode 100644 sqlmesh_dbt/__init__.py create mode 100644 sqlmesh_dbt/cli.py create mode 100644 sqlmesh_dbt/console.py create mode 100644 sqlmesh_dbt/operations.py create mode 100644 tests/dbt/cli/__init__.py create mode 100644 tests/dbt/cli/conftest.py create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/dbt_project.yml create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/customers.sql create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/docs.md create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/orders.sql create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/overview.md create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/schema.yml create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/schema.yml create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_customers.sql create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_orders.sql create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_payments.sql create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/profiles.yml create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/.gitkeep create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_customers.csv create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_orders.csv create mode 100644 tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_payments.csv create mode 100644 tests/dbt/cli/test_list.py create mode 100644 tests/dbt/cli/test_operations.py create mode 100644 tests/dbt/cli/test_run.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e4b61cb0f3..bb63cf1be1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: language: python types_or: [python, pyi] require_serial: true - files: &files ^(sqlmesh/|tests/|web/|examples/|setup.py) + files: &files ^(sqlmesh/|sqlmesh_dbt/|tests/|web/|examples/|setup.py) - id: ruff-format name: ruff-format entry: ruff format --force-exclude --line-length 100 diff --git a/pyproject.toml b/pyproject.toml index 8e1c4b879f..517f3be426 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,6 +142,7 @@ risingwave = ["psycopg2"] [project.scripts] sqlmesh = "sqlmesh.cli.main:cli" +sqlmesh_dbt = "sqlmesh_dbt.cli:dbt" sqlmesh_cicd = "sqlmesh.cicd.bot:bot" sqlmesh_lsp = "sqlmesh.lsp.main:main" @@ -164,7 +165,7 @@ fallback_version = "0.0.0" local_scheme = "no-local-version" [tool.setuptools.packages.find] -include = ["sqlmesh", "sqlmesh.*", "web*"] +include = ["sqlmesh", "sqlmesh.*", "sqlmesh_dbt", "sqlmesh_dbt.*", "web*"] [tool.setuptools.package-data] web = ["client/dist/**"] diff --git a/sqlmesh_dbt/__init__.py b/sqlmesh_dbt/__init__.py new file mode 100644 index 0000000000..984f083f5b --- /dev/null +++ b/sqlmesh_dbt/__init__.py @@ -0,0 +1,5 @@ +# Note: `sqlmesh_dbt` is deliberately in its own package from `sqlmesh` to avoid the upfront time overhead +# that comes from `import sqlmesh` +# +# Obviously we still have to `import sqlmesh` at some point but this allows us to defer it until needed, +# which means we can make the CLI feel more responsive by being able to output something immediately diff --git a/sqlmesh_dbt/cli.py b/sqlmesh_dbt/cli.py new file mode 100644 index 0000000000..2ec59b665d --- /dev/null +++ b/sqlmesh_dbt/cli.py @@ -0,0 +1,80 @@ +import typing as t +import sys +import click +from sqlmesh_dbt.operations import DbtOperations, create + + +def _get_dbt_operations(ctx: click.Context) -> DbtOperations: + if not isinstance(ctx.obj, DbtOperations): + raise ValueError(f"Unexpected click context object: {type(ctx.obj)}") + return ctx.obj + + +@click.group() +@click.pass_context +def dbt(ctx: click.Context) -> None: + """ + An ELT tool for managing your SQL transformations and data models, powered by the SQLMesh engine. + """ + + if "--help" in sys.argv: + # we dont need to import sqlmesh/load the project for CLI help + return + + # TODO: conditionally call create() if there are times we dont want/need to import sqlmesh and load a project + ctx.obj = create() + + +@dbt.command() +@click.option("-s", "-m", "--select", "--models", "--model", help="Specify the nodes to include.") +@click.option( + "-f", + "--full-refresh", + help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", +) +@click.pass_context +def run(ctx: click.Context, select: t.Optional[str], full_refresh: bool) -> None: + """Compile SQL and execute against the current target database.""" + _get_dbt_operations(ctx).run(select=select, full_refresh=full_refresh) + + +@dbt.command(name="list") +@click.pass_context +def list_(ctx: click.Context) -> None: + """List the resources in your project""" + _get_dbt_operations(ctx).list_() + + +@dbt.command(name="ls", hidden=True) # hidden alias for list +@click.pass_context +def ls(ctx: click.Context) -> None: + """List the resources in your project""" + ctx.forward(list_) + + +def _not_implemented(name: str) -> None: + @dbt.command(name=name) + def _not_implemented() -> None: + """Not implemented""" + click.echo(f"dbt {name} not implemented") + + +for subcommand in ( + "build", + "clean", + "clone", + "compile", + "debug", + "deps", + "docs", + "init", + "parse", + "retry", + "run-operation", + "seed", + "show", + "snapshot", + "source", + "test", +): + _not_implemented(subcommand) diff --git a/sqlmesh_dbt/console.py b/sqlmesh_dbt/console.py new file mode 100644 index 0000000000..7d804ceb71 --- /dev/null +++ b/sqlmesh_dbt/console.py @@ -0,0 +1,8 @@ +from sqlmesh.core.console import TerminalConsole + + +class DbtCliConsole(TerminalConsole): + # TODO: build this out + + def print(self, msg: str) -> None: + return self._print(msg) diff --git a/sqlmesh_dbt/operations.py b/sqlmesh_dbt/operations.py new file mode 100644 index 0000000000..ba666abf09 --- /dev/null +++ b/sqlmesh_dbt/operations.py @@ -0,0 +1,133 @@ +from __future__ import annotations +import typing as t +from rich.progress import Progress +from pathlib import Path + +if t.TYPE_CHECKING: + # important to gate these to be able to defer importing sqlmesh until we need to + from sqlmesh.core.context import Context + from sqlmesh.dbt.project import Project + from sqlmesh_dbt.console import DbtCliConsole + + +class DbtOperations: + def __init__(self, sqlmesh_context: Context, dbt_project: Project): + self.context = sqlmesh_context + self.project = dbt_project + + def list_(self) -> None: + for _, model in self.context.models.items(): + self.console.print(model.name) + + def run(self, select: t.Optional[str] = None, full_refresh: bool = False) -> None: + # A dbt run both updates data and changes schemas and has no way of rolling back so more closely maps to a SQLMesh forward-only plan + # TODO: if --full-refresh specified, mark incrementals as breaking instead of forward_only? + + # TODO: we need to either convert DBT selector syntax to SQLMesh selector syntax + # or make the model selection engine configurable + select_models = None + if select: + if "," in select: + select_models = select.split(",") + else: + select_models = select.split(" ") + + self.context.plan( + select_models=select_models, + forward_only=True, + no_auto_categorization=True, # everything is breaking / foward-only + effective_from=self.context.config.model_defaults.start, + run=True, + auto_apply=True, + ) + + @property + def console(self) -> DbtCliConsole: + console = self.context.console + from sqlmesh_dbt.console import DbtCliConsole + + if not isinstance(console, DbtCliConsole): + raise ValueError(f"Expecting dbt cli console, got: {console}") + + return console + + +def create( + project_dir: t.Optional[Path] = None, profiles_dir: t.Optional[Path] = None, debug: bool = False +) -> DbtOperations: + with Progress(transient=True) as progress: + # Indeterminate progress bar before SQLMesh import to provide feedback to the user that something is indeed happening + load_task_id = progress.add_task("Loading engine", total=None) + + from sqlmesh import configure_logging + from sqlmesh.core.context import Context + from sqlmesh.dbt.loader import sqlmesh_config, DbtLoader + from sqlmesh.core.console import set_console + from sqlmesh_dbt.console import DbtCliConsole + from sqlmesh.utils.errors import SQLMeshError + + configure_logging(force_debug=debug) + set_console(DbtCliConsole()) + + progress.update(load_task_id, description="Loading project", total=None) + + # inject default start date if one is not specified to prevent the user from having to do anything + _inject_default_start_date(project_dir) + + config = sqlmesh_config( + project_root=project_dir, + # do we want to use a local duckdb for state? + # warehouse state has a bunch of overhead to initialize, is slow for ongoing operations and will create tables that perhaps the user was not expecting + # on the other hand, local state is not portable + state_connection=None, + ) + + sqlmesh_context = Context( + config=config, + load=True, + ) + + # this helps things which want a default project-level start date, like the "effective from date" for forward-only plans + if not sqlmesh_context.config.model_defaults.start: + min_start_date = min( + ( + model.start + for model in sqlmesh_context.models.values() + if model.start is not None + ), + default=None, + ) + sqlmesh_context.config.model_defaults.start = min_start_date + + dbt_loader = sqlmesh_context._loaders[0] + if not isinstance(dbt_loader, DbtLoader): + raise SQLMeshError(f"Unexpected loader type: {type(dbt_loader)}") + + # so that DbtOperations can query information from the DBT project files in order to invoke SQLMesh correctly + dbt_project = dbt_loader._projects[0] + + return DbtOperations(sqlmesh_context, dbt_project) + + +def _inject_default_start_date(project_dir: t.Optional[Path] = None) -> None: + """ + SQLMesh needs a start date to as the starting point for calculating intervals on incremental models + + Rather than forcing the user to update their config manually or having a default that is not saved between runs, + we can inject it automatically to the dbt_project.yml file + """ + from sqlmesh.dbt.project import PROJECT_FILENAME, load_yaml + from sqlmesh.utils.yaml import dump + from sqlmesh.utils.date import yesterday_ds + + project_yaml_path = (project_dir or Path.cwd()) / PROJECT_FILENAME + if project_yaml_path.exists(): + loaded_project_file = load_yaml(project_yaml_path) + start_date_keys = ("start", "+start") + if "models" in loaded_project_file and all( + k not in loaded_project_file["models"] for k in start_date_keys + ): + loaded_project_file["models"]["+start"] = yesterday_ds() + # todo: this may format the file differently, is that acceptable? + with project_yaml_path.open("w") as f: + dump(loaded_project_file, f) diff --git a/tests/dbt/cli/__init__.py b/tests/dbt/cli/__init__.py new file mode 100644 index 0000000000..f44140c36b --- /dev/null +++ b/tests/dbt/cli/__init__.py @@ -0,0 +1 @@ +pytestmark = ["foo"] diff --git a/tests/dbt/cli/conftest.py b/tests/dbt/cli/conftest.py new file mode 100644 index 0000000000..dfad2f0046 --- /dev/null +++ b/tests/dbt/cli/conftest.py @@ -0,0 +1,29 @@ +import typing as t +from pathlib import Path +import os +import functools +from click.testing import CliRunner, Result +import pytest + + +@pytest.fixture +def jaffle_shop_duckdb(copy_to_temp_path: t.Callable[..., t.List[Path]]) -> t.Iterable[Path]: + fixture_path = Path(__file__).parent / "fixtures" / "jaffle_shop_duckdb" + assert fixture_path.exists() + + current_path = os.getcwd() + output_path = copy_to_temp_path(paths=fixture_path)[0] + + # so that we can invoke commands from the perspective of a user that is alrady in the correct directory + os.chdir(output_path) + + yield output_path + + os.chdir(current_path) + + +@pytest.fixture +def invoke_cli() -> t.Callable[..., Result]: + from sqlmesh_dbt.cli import dbt + + return functools.partial(CliRunner().invoke, dbt) diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/dbt_project.yml b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/dbt_project.yml new file mode 100644 index 0000000000..1b71726467 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/dbt_project.yml @@ -0,0 +1,34 @@ +name: 'jaffle_shop' + +config-version: 2 +version: '0.1' + +profile: 'jaffle_shop' + +model-paths: ["models"] +seed-paths: ["seeds"] +test-paths: ["tests"] +analysis-paths: ["analysis"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_modules" + - "logs" + +require-dbt-version: [">=1.0.0", "<2.0.0"] + +seeds: + +docs: + node_color: '#cd7f32' + +models: + jaffle_shop: + +materialized: table + staging: + +materialized: view + +docs: + node_color: 'silver' + +docs: + node_color: 'gold' diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/customers.sql b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/customers.sql new file mode 100644 index 0000000000..016a004fe5 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/customers.sql @@ -0,0 +1,69 @@ +with customers as ( + + select * from {{ ref('stg_customers') }} + +), + +orders as ( + + select * from {{ ref('stg_orders') }} + +), + +payments as ( + + select * from {{ ref('stg_payments') }} + +), + +customer_orders as ( + + select + customer_id, + + min(order_date) as first_order, + max(order_date) as most_recent_order, + count(order_id) as number_of_orders + from orders + + group by customer_id + +), + +customer_payments as ( + + select + orders.customer_id, + sum(amount) as total_amount + + from payments + + left join orders on + payments.order_id = orders.order_id + + group by orders.customer_id + +), + +final as ( + + select + customers.customer_id, + customers.first_name, + customers.last_name, + customer_orders.first_order, + customer_orders.most_recent_order, + customer_orders.number_of_orders, + customer_payments.total_amount as customer_lifetime_value + + from customers + + left join customer_orders + on customers.customer_id = customer_orders.customer_id + + left join customer_payments + on customers.customer_id = customer_payments.customer_id + +) + +select * from final diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/docs.md b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/docs.md new file mode 100644 index 0000000000..c6ae93be07 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/docs.md @@ -0,0 +1,14 @@ +{% docs orders_status %} + +Orders can be one of the following statuses: + +| status | description | +|----------------|------------------------------------------------------------------------------------------------------------------------| +| placed | The order has been placed but has not yet left the warehouse | +| shipped | The order has ben shipped to the customer and is currently in transit | +| completed | The order has been received by the customer | +| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse | +| returned | The order has been returned by the customer and received at the warehouse | + + +{% enddocs %} diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/orders.sql b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/orders.sql new file mode 100644 index 0000000000..cbb2934911 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/orders.sql @@ -0,0 +1,56 @@ +{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %} + +with orders as ( + + select * from {{ ref('stg_orders') }} + +), + +payments as ( + + select * from {{ ref('stg_payments') }} + +), + +order_payments as ( + + select + order_id, + + {% for payment_method in payment_methods -%} + sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount, + {% endfor -%} + + sum(amount) as total_amount + + from payments + + group by order_id + +), + +final as ( + + select + orders.order_id, + orders.customer_id, + orders.order_date, + orders.status, + + {% for payment_method in payment_methods -%} + + order_payments.{{ payment_method }}_amount, + + {% endfor -%} + + order_payments.total_amount as amount + + from orders + + + left join order_payments + on orders.order_id = order_payments.order_id + +) + +select * from final diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/overview.md b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/overview.md new file mode 100644 index 0000000000..0544c42b17 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/overview.md @@ -0,0 +1,11 @@ +{% docs __overview__ %} + +## Data Documentation for Jaffle Shop + +`jaffle_shop` is a fictional ecommerce store. + +This [dbt](https://www.getdbt.com/) project is for testing out code. + +The source code can be found [here](https://github.com/clrcrl/jaffle_shop). + +{% enddocs %} diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/schema.yml b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/schema.yml new file mode 100644 index 0000000000..381349cfda --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/schema.yml @@ -0,0 +1,82 @@ +version: 2 + +models: + - name: customers + description: This table has basic information about a customer, as well as some derived facts based on a customer's orders + + columns: + - name: customer_id + description: This is a unique identifier for a customer + tests: + - unique + - not_null + + - name: first_name + description: Customer's first name. PII. + + - name: last_name + description: Customer's last name. PII. + + - name: first_order + description: Date (UTC) of a customer's first order + + - name: most_recent_order + description: Date (UTC) of a customer's most recent order + + - name: number_of_orders + description: Count of the number of orders a customer has placed + + - name: total_order_amount + description: Total value (AUD) of a customer's orders + + - name: orders + description: This table has basic information about orders, as well as some derived facts based on payments + + columns: + - name: order_id + tests: + - unique + - not_null + description: This is a unique identifier for an order + + - name: customer_id + description: Foreign key to the customers table + tests: + - not_null + - relationships: + to: ref('customers') + field: customer_id + + - name: order_date + description: Date (UTC) that the order was placed + + - name: status + description: '{{ doc("orders_status") }}' + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: amount + description: Total amount (AUD) of the order + tests: + - not_null + + - name: credit_card_amount + description: Amount of the order (AUD) paid for by credit card + tests: + - not_null + + - name: coupon_amount + description: Amount of the order (AUD) paid for by coupon + tests: + - not_null + + - name: bank_transfer_amount + description: Amount of the order (AUD) paid for by bank transfer + tests: + - not_null + + - name: gift_card_amount + description: Amount of the order (AUD) paid for by gift card + tests: + - not_null diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/schema.yml b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/schema.yml new file mode 100644 index 0000000000..c207e4cf52 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/schema.yml @@ -0,0 +1,31 @@ +version: 2 + +models: + - name: stg_customers + columns: + - name: customer_id + tests: + - unique + - not_null + + - name: stg_orders + columns: + - name: order_id + tests: + - unique + - not_null + - name: status + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: stg_payments + columns: + - name: payment_id + tests: + - unique + - not_null + - name: payment_method + tests: + - accepted_values: + values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_customers.sql b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_customers.sql new file mode 100644 index 0000000000..cad0472695 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_customers.sql @@ -0,0 +1,22 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_customers') }} + +), + +renamed as ( + + select + id as customer_id, + first_name, + last_name + + from source + +) + +select * from renamed diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_orders.sql b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_orders.sql new file mode 100644 index 0000000000..a654dcb947 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_orders.sql @@ -0,0 +1,23 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_orders') }} + +), + +renamed as ( + + select + id as order_id, + user_id as customer_id, + order_date, + status + + from source + +) + +select * from renamed diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_payments.sql b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_payments.sql new file mode 100644 index 0000000000..700cf7f4f6 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/models/staging/stg_payments.sql @@ -0,0 +1,25 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_payments') }} + +), + +renamed as ( + + select + id as payment_id, + order_id, + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars + amount / 100 as amount + + from source + +) + +select * from renamed diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/profiles.yml b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/profiles.yml new file mode 100644 index 0000000000..9008a2d62c --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/profiles.yml @@ -0,0 +1,8 @@ +jaffle_shop: + + target: dev + outputs: + dev: + type: duckdb + path: 'jaffle_shop.duckdb' + threads: 24 diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/.gitkeep b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_customers.csv b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_customers.csv new file mode 100644 index 0000000000..b3e6747d69 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_customers.csv @@ -0,0 +1,101 @@ +id,first_name,last_name +1,Michael,P. +2,Shawn,M. +3,Kathleen,P. +4,Jimmy,C. +5,Katherine,R. +6,Sarah,R. +7,Martin,M. +8,Frank,R. +9,Jennifer,F. +10,Henry,W. +11,Fred,S. +12,Amy,D. +13,Kathleen,M. +14,Steve,F. +15,Teresa,H. +16,Amanda,H. +17,Kimberly,R. +18,Johnny,K. +19,Virginia,F. +20,Anna,A. +21,Willie,H. +22,Sean,H. +23,Mildred,A. +24,David,G. +25,Victor,H. +26,Aaron,R. +27,Benjamin,B. +28,Lisa,W. +29,Benjamin,K. +30,Christina,W. +31,Jane,G. +32,Thomas,O. +33,Katherine,M. +34,Jennifer,S. +35,Sara,T. +36,Harold,O. +37,Shirley,J. +38,Dennis,J. +39,Louise,W. +40,Maria,A. +41,Gloria,C. +42,Diana,S. +43,Kelly,N. +44,Jane,R. +45,Scott,B. +46,Norma,C. +47,Marie,P. +48,Lillian,C. +49,Judy,N. +50,Billy,L. +51,Howard,R. +52,Laura,F. +53,Anne,B. +54,Rose,M. +55,Nicholas,R. +56,Joshua,K. +57,Paul,W. +58,Kathryn,K. +59,Adam,A. +60,Norma,W. +61,Timothy,R. +62,Elizabeth,P. +63,Edward,G. +64,David,C. +65,Brenda,W. +66,Adam,W. +67,Michael,H. +68,Jesse,E. +69,Janet,P. +70,Helen,F. +71,Gerald,C. +72,Kathryn,O. +73,Alan,B. +74,Harry,A. +75,Andrea,H. +76,Barbara,W. +77,Anne,W. +78,Harry,H. +79,Jack,R. +80,Phillip,H. +81,Shirley,H. +82,Arthur,D. +83,Virginia,R. +84,Christina,R. +85,Theresa,M. +86,Jason,C. +87,Phillip,B. +88,Adam,T. +89,Margaret,J. +90,Paul,P. +91,Todd,W. +92,Willie,O. +93,Frances,R. +94,Gregory,H. +95,Lisa,P. +96,Jacqueline,A. +97,Shirley,D. +98,Nicole,M. +99,Mary,G. +100,Jean,M. diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_orders.csv b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_orders.csv new file mode 100644 index 0000000000..7c2be07888 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_orders.csv @@ -0,0 +1,100 @@ +id,user_id,order_date,status +1,1,2018-01-01,returned +2,3,2018-01-02,completed +3,94,2018-01-04,completed +4,50,2018-01-05,completed +5,64,2018-01-05,completed +6,54,2018-01-07,completed +7,88,2018-01-09,completed +8,2,2018-01-11,returned +9,53,2018-01-12,completed +10,7,2018-01-14,completed +11,99,2018-01-14,completed +12,59,2018-01-15,completed +13,84,2018-01-17,completed +14,40,2018-01-17,returned +15,25,2018-01-17,completed +16,39,2018-01-18,completed +17,71,2018-01-18,completed +18,64,2018-01-20,returned +19,54,2018-01-22,completed +20,20,2018-01-23,completed +21,71,2018-01-23,completed +22,86,2018-01-24,completed +23,22,2018-01-26,return_pending +24,3,2018-01-27,completed +25,51,2018-01-28,completed +26,32,2018-01-28,completed +27,94,2018-01-29,completed +28,8,2018-01-29,completed +29,57,2018-01-31,completed +30,69,2018-02-02,completed +31,16,2018-02-02,completed +32,28,2018-02-04,completed +33,42,2018-02-04,completed +34,38,2018-02-06,completed +35,80,2018-02-08,completed +36,85,2018-02-10,completed +37,1,2018-02-10,completed +38,51,2018-02-10,completed +39,26,2018-02-11,completed +40,33,2018-02-13,completed +41,99,2018-02-14,completed +42,92,2018-02-16,completed +43,31,2018-02-17,completed +44,66,2018-02-17,completed +45,22,2018-02-17,completed +46,6,2018-02-19,completed +47,50,2018-02-20,completed +48,27,2018-02-21,completed +49,35,2018-02-21,completed +50,51,2018-02-23,completed +51,71,2018-02-24,completed +52,54,2018-02-25,return_pending +53,34,2018-02-26,completed +54,54,2018-02-26,completed +55,18,2018-02-27,completed +56,79,2018-02-28,completed +57,93,2018-03-01,completed +58,22,2018-03-01,completed +59,30,2018-03-02,completed +60,12,2018-03-03,completed +61,63,2018-03-03,completed +62,57,2018-03-05,completed +63,70,2018-03-06,completed +64,13,2018-03-07,completed +65,26,2018-03-08,completed +66,36,2018-03-10,completed +67,79,2018-03-11,completed +68,53,2018-03-11,completed +69,3,2018-03-11,completed +70,8,2018-03-12,completed +71,42,2018-03-12,shipped +72,30,2018-03-14,shipped +73,19,2018-03-16,completed +74,9,2018-03-17,shipped +75,69,2018-03-18,completed +76,25,2018-03-20,completed +77,35,2018-03-21,shipped +78,90,2018-03-23,shipped +79,52,2018-03-23,shipped +80,11,2018-03-23,shipped +81,76,2018-03-23,shipped +82,46,2018-03-24,shipped +83,54,2018-03-24,shipped +84,70,2018-03-26,placed +85,47,2018-03-26,shipped +86,68,2018-03-26,placed +87,46,2018-03-27,placed +88,91,2018-03-27,shipped +89,21,2018-03-28,placed +90,66,2018-03-30,shipped +91,47,2018-03-31,placed +92,84,2018-04-02,placed +93,66,2018-04-03,placed +94,63,2018-04-03,placed +95,27,2018-04-04,placed +96,90,2018-04-06,placed +97,89,2018-04-07,placed +98,41,2018-04-07,placed +99,85,2018-04-09,placed diff --git a/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_payments.csv b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_payments.csv new file mode 100644 index 0000000000..a587baab59 --- /dev/null +++ b/tests/dbt/cli/fixtures/jaffle_shop_duckdb/seeds/raw_payments.csv @@ -0,0 +1,114 @@ +id,order_id,payment_method,amount +1,1,credit_card,1000 +2,2,credit_card,2000 +3,3,coupon,100 +4,4,coupon,2500 +5,5,bank_transfer,1700 +6,6,credit_card,600 +7,7,credit_card,1600 +8,8,credit_card,2300 +9,9,gift_card,2300 +10,9,bank_transfer,0 +11,10,bank_transfer,2600 +12,11,credit_card,2700 +13,12,credit_card,100 +14,13,credit_card,500 +15,13,bank_transfer,1400 +16,14,bank_transfer,300 +17,15,coupon,2200 +18,16,credit_card,1000 +19,17,bank_transfer,200 +20,18,credit_card,500 +21,18,credit_card,800 +22,19,gift_card,600 +23,20,bank_transfer,1500 +24,21,credit_card,1200 +25,22,bank_transfer,800 +26,23,gift_card,2300 +27,24,coupon,2600 +28,25,bank_transfer,2000 +29,25,credit_card,2200 +30,25,coupon,1600 +31,26,credit_card,3000 +32,27,credit_card,2300 +33,28,bank_transfer,1900 +34,29,bank_transfer,1200 +35,30,credit_card,1300 +36,31,credit_card,1200 +37,32,credit_card,300 +38,33,credit_card,2200 +39,34,bank_transfer,1500 +40,35,credit_card,2900 +41,36,bank_transfer,900 +42,37,credit_card,2300 +43,38,credit_card,1500 +44,39,bank_transfer,800 +45,40,credit_card,1400 +46,41,credit_card,1700 +47,42,coupon,1700 +48,43,gift_card,1800 +49,44,gift_card,1100 +50,45,bank_transfer,500 +51,46,bank_transfer,800 +52,47,credit_card,2200 +53,48,bank_transfer,300 +54,49,credit_card,600 +55,49,credit_card,900 +56,50,credit_card,2600 +57,51,credit_card,2900 +58,51,credit_card,100 +59,52,bank_transfer,1500 +60,53,credit_card,300 +61,54,credit_card,1800 +62,54,bank_transfer,1100 +63,55,credit_card,2900 +64,56,credit_card,400 +65,57,bank_transfer,200 +66,58,coupon,1800 +67,58,gift_card,600 +68,59,gift_card,2800 +69,60,credit_card,400 +70,61,bank_transfer,1600 +71,62,gift_card,1400 +72,63,credit_card,2900 +73,64,bank_transfer,2600 +74,65,credit_card,0 +75,66,credit_card,2800 +76,67,bank_transfer,400 +77,67,credit_card,1900 +78,68,credit_card,1600 +79,69,credit_card,1900 +80,70,credit_card,2600 +81,71,credit_card,500 +82,72,credit_card,2900 +83,73,bank_transfer,300 +84,74,credit_card,3000 +85,75,credit_card,1900 +86,76,coupon,200 +87,77,credit_card,0 +88,77,bank_transfer,1900 +89,78,bank_transfer,2600 +90,79,credit_card,1800 +91,79,credit_card,900 +92,80,gift_card,300 +93,81,coupon,200 +94,82,credit_card,800 +95,83,credit_card,100 +96,84,bank_transfer,2500 +97,85,bank_transfer,1700 +98,86,coupon,2300 +99,87,gift_card,3000 +100,87,credit_card,2600 +101,88,credit_card,2900 +102,89,bank_transfer,2200 +103,90,bank_transfer,200 +104,91,credit_card,1900 +105,92,bank_transfer,1500 +106,92,coupon,200 +107,93,gift_card,2600 +108,94,coupon,700 +109,95,coupon,2400 +110,96,gift_card,1700 +111,97,bank_transfer,1400 +112,98,bank_transfer,1000 +113,99,credit_card,2400 diff --git a/tests/dbt/cli/test_list.py b/tests/dbt/cli/test_list.py new file mode 100644 index 0000000000..9312be8635 --- /dev/null +++ b/tests/dbt/cli/test_list.py @@ -0,0 +1,17 @@ +import typing as t +import pytest +from pathlib import Path +from click.testing import Result + +pytestmark = pytest.mark.slow + + +def test_list(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Result]): + result = invoke_cli(["list"]) + + assert result.exit_code == 0 + assert not result.exception + + assert "main.orders" in result.output + assert "main.customers" in result.output + assert "main.stg_payments" in result.output diff --git a/tests/dbt/cli/test_operations.py b/tests/dbt/cli/test_operations.py new file mode 100644 index 0000000000..e384028bbc --- /dev/null +++ b/tests/dbt/cli/test_operations.py @@ -0,0 +1,57 @@ +from pathlib import Path +import pytest +from sqlmesh_dbt.operations import create +from sqlmesh.utils import yaml +import time_machine + +pytestmark = pytest.mark.slow + + +def test_create_injects_default_start_date(jaffle_shop_duckdb: Path): + with time_machine.travel("2020-01-02 00:00:00 UTC"): + from sqlmesh.utils.date import yesterday_ds + + assert yesterday_ds() == "2020-01-01" + + operations = create() + + assert operations.context.config.model_defaults.start == "2020-01-01" + assert all( + model.start == "2020-01-01" + for model in operations.context.models.values() + if not model.kind.is_seed + ) + + # check that the date set on the first invocation persists to future invocations + from sqlmesh.utils.date import yesterday_ds + + assert yesterday_ds() != "2020-01-01" + + operations = create() + + assert operations.context.config.model_defaults.start == "2020-01-01" + assert all( + model.start == "2020-01-01" + for model in operations.context.models.values() + if not model.kind.is_seed + ) + + +def test_create_uses_configured_start_date_if_supplied(jaffle_shop_duckdb: Path): + dbt_project_yaml = jaffle_shop_duckdb / "dbt_project.yml" + + contents = yaml.load(dbt_project_yaml, render_jinja=False) + + contents["models"]["+start"] = "2023-12-12" + + with dbt_project_yaml.open("w") as f: + yaml.dump(contents, f) + + operations = create() + + assert operations.context.config.model_defaults.start == "2023-12-12" + assert all( + model.start == "2023-12-12" + for model in operations.context.models.values() + if not model.kind.is_seed + ) diff --git a/tests/dbt/cli/test_run.py b/tests/dbt/cli/test_run.py new file mode 100644 index 0000000000..0e4a04bcb1 --- /dev/null +++ b/tests/dbt/cli/test_run.py @@ -0,0 +1,15 @@ +import typing as t +import pytest +from pathlib import Path +from click.testing import Result + +pytestmark = pytest.mark.slow + + +def test_run(jaffle_shop_duckdb: Path, invoke_cli: t.Callable[..., Result]): + result = invoke_cli(["run"]) + + assert result.exit_code == 0 + assert not result.exception + + assert "Model batches executed" in result.output From 110a9c677042126ebe8be0eda142b5d8a7820118 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Mon, 11 Aug 2025 22:38:15 +0000 Subject: [PATCH 2/2] PR feedback --- sqlmesh_dbt/operations.py | 4 ++-- tests/dbt/cli/__init__.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 tests/dbt/cli/__init__.py diff --git a/sqlmesh_dbt/operations.py b/sqlmesh_dbt/operations.py index ba666abf09..b826a00e37 100644 --- a/sqlmesh_dbt/operations.py +++ b/sqlmesh_dbt/operations.py @@ -34,10 +34,10 @@ def run(self, select: t.Optional[str] = None, full_refresh: bool = False) -> Non self.context.plan( select_models=select_models, - forward_only=True, no_auto_categorization=True, # everything is breaking / foward-only - effective_from=self.context.config.model_defaults.start, run=True, + no_diff=True, + no_prompts=True, auto_apply=True, ) diff --git a/tests/dbt/cli/__init__.py b/tests/dbt/cli/__init__.py deleted file mode 100644 index f44140c36b..0000000000 --- a/tests/dbt/cli/__init__.py +++ /dev/null @@ -1 +0,0 @@ -pytestmark = ["foo"]