From 52a6400213876e092e2a953d9c4dba8e0395012d Mon Sep 17 00:00:00 2001 From: eakmanrq <6326532+eakmanrq@users.noreply.github.com> Date: Tue, 9 Sep 2025 13:31:04 -0700 Subject: [PATCH] fix: dbt time column serialization --- sqlmesh/dbt/model.py | 4 +--- tests/dbt/test_manifest.py | 9 ++++----- tests/dbt/test_model.py | 11 +++++++++-- .../sushi_test/models/waiter_as_customer_by_day.sql | 2 +- .../dbt/sushi_test/models/waiter_revenue_by_day.sql | 2 +- .../sushi_test/models/waiter_revenue_by_day_v1.sql | 2 +- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/sqlmesh/dbt/model.py b/sqlmesh/dbt/model.py index a4ebf93ae5..84a51f55a2 100644 --- a/sqlmesh/dbt/model.py +++ b/sqlmesh/dbt/model.py @@ -31,7 +31,6 @@ OnAdditiveChange, on_destructive_change_validator, on_additive_change_validator, - TimeColumn, ) from sqlmesh.dbt.basemodel import BaseModelConfig, Materialization, SnapshotStrategy from sqlmesh.dbt.common import SqlStr, sql_str_validator @@ -86,7 +85,7 @@ class ModelConfig(BaseModelConfig): # sqlmesh fields sql: SqlStr = SqlStr("") - time_column: t.Optional[TimeColumn] = None + time_column: t.Optional[t.Union[str, t.Dict[str, str]]] = None cron: t.Optional[str] = None interval_unit: t.Optional[str] = None batch_concurrency: t.Optional[int] = None @@ -153,7 +152,6 @@ class ModelConfig(BaseModelConfig): _sql_validator = sql_str_validator _on_destructive_change_validator = on_destructive_change_validator _on_additive_change_validator = on_additive_change_validator - _time_column_validator = TimeColumn.validator() @field_validator( "unique_key", diff --git a/tests/dbt/test_manifest.py b/tests/dbt/test_manifest.py index 1ea94cceb0..e5e98eae49 100644 --- a/tests/dbt/test_manifest.py +++ b/tests/dbt/test_manifest.py @@ -5,7 +5,6 @@ import pytest from sqlmesh.core.config import ModelDefaultsConfig -from sqlmesh.core.model import TimeColumn from sqlmesh.dbt.basemodel import Dependencies from sqlmesh.dbt.common import ModelAttrs from sqlmesh.dbt.context import DbtContext @@ -82,9 +81,9 @@ def test_manifest_helper(caplog): macros=[MacroReference(name="ref")], ) assert waiter_as_customer_by_day_config.materialized == "incremental" - assert waiter_as_customer_by_day_config.incremental_strategy == "delete+insert" + assert waiter_as_customer_by_day_config.incremental_strategy == "incremental_by_time_range" assert waiter_as_customer_by_day_config.cluster_by == ["ds"] - assert waiter_as_customer_by_day_config.time_column == TimeColumn.create("ds", "duckdb") + assert waiter_as_customer_by_day_config.time_column == "ds" if DBT_VERSION >= (1, 5, 0): waiter_revenue_by_day_config = models["waiter_revenue_by_day_v2"] @@ -104,9 +103,9 @@ def test_manifest_helper(caplog): has_dynamic_var_names=True, ) assert waiter_revenue_by_day_config.materialized == "incremental" - assert waiter_revenue_by_day_config.incremental_strategy == "delete+insert" + assert waiter_revenue_by_day_config.incremental_strategy == "incremental_by_time_range" assert waiter_revenue_by_day_config.cluster_by == ["ds"] - assert waiter_revenue_by_day_config.time_column == TimeColumn.create("ds", "duckdb") + assert waiter_revenue_by_day_config.time_column == "ds" assert waiter_revenue_by_day_config.dialect_ == "bigquery" assert helper.models("customers")["customers"].dependencies == Dependencies( diff --git a/tests/dbt/test_model.py b/tests/dbt/test_model.py index bfc18144ef..ffdba2ae05 100644 --- a/tests/dbt/test_model.py +++ b/tests/dbt/test_model.py @@ -9,6 +9,7 @@ from sqlmesh import Context from sqlmesh.core.model import TimeColumn, IncrementalByTimeRangeKind from sqlmesh.core.model.kind import OnDestructiveChange, OnAdditiveChange +from sqlmesh.core.state_sync.db.snapshot import _snapshot_to_json from sqlmesh.dbt.common import Dependencies from sqlmesh.dbt.context import DbtContext from sqlmesh.dbt.model import ModelConfig @@ -328,7 +329,8 @@ def test_load_incremental_time_range_strategy_required_only( snapshot_fqn = '"local"."main"."incremental_time_range"' context = Context(paths=project_dir) - model = context.snapshots[snapshot_fqn].model + snapshot = context.snapshots[snapshot_fqn] + model = snapshot.model # Validate model-level attributes assert model.start == "2025-01-01" assert model.interval_unit.is_day @@ -342,6 +344,8 @@ def test_load_incremental_time_range_strategy_required_only( assert model.depends_on_self is False assert model.kind.auto_restatement_intervals is None assert model.kind.partition_by_time_column is True + # make sure the snapshot can be serialized to json + assert isinstance(_snapshot_to_json(snapshot), str) @pytest.mark.slow @@ -381,7 +385,8 @@ def test_load_incremental_time_range_strategy_all_defined( snapshot_fqn = '"local"."main"."incremental_time_range"' context = Context(paths=project_dir) - model = context.snapshots[snapshot_fqn].model + snapshot = context.snapshots[snapshot_fqn] + model = snapshot.model # Validate model-level attributes assert model.start == "2025-01-01" assert model.interval_unit.is_day @@ -402,6 +407,8 @@ def test_load_incremental_time_range_strategy_all_defined( assert model.kind.batch_size == 3 assert model.kind.batch_concurrency == 2 assert model.depends_on_self is False + # make sure the snapshot can be serialized to json + assert isinstance(_snapshot_to_json(snapshot), str) @pytest.mark.slow diff --git a/tests/fixtures/dbt/sushi_test/models/waiter_as_customer_by_day.sql b/tests/fixtures/dbt/sushi_test/models/waiter_as_customer_by_day.sql index 82237634b5..ed845b67cb 100644 --- a/tests/fixtures/dbt/sushi_test/models/waiter_as_customer_by_day.sql +++ b/tests/fixtures/dbt/sushi_test/models/waiter_as_customer_by_day.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - incremental_strategy='delete+insert', + incremental_strategy='incremental_by_time_range', cluster_by=['ds'], time_column='ds', ) diff --git a/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day.sql b/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day.sql index 5eeb0002e0..2731b07019 100644 --- a/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day.sql +++ b/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - incremental_strategy='delete+insert', + incremental_strategy='incremental_by_time_range', cluster_by=['ds'], time_column='ds', dialect="bigquery" diff --git a/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day_v1.sql b/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day_v1.sql index 335e7ab799..e229dc8b91 100644 --- a/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day_v1.sql +++ b/tests/fixtures/dbt/sushi_test/models/waiter_revenue_by_day_v1.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - incremental_strategy='delete+insert', + incremental_strategy='incremental_by_time_range', cluster_by=['ds'], time_column='ds', dialect="bigquery"