Skip to content

Commit b08a8ce

Browse files
committed
Chore!: add migration script to warn about dbt data_type-related diffs
1 parent db58405 commit b08a8ce

File tree

1 file changed

+43
-0
lines changed

1 file changed

+43
-0
lines changed
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
"""
2+
Warns dbt users about potential diffs due to corrected data_type handling.
3+
4+
SQLMesh previously treated dbt's schema.yml data_type field as columns_to_types, which
5+
doesn't match dbt's behavior. dbt only uses data_type for contracts/validation, not DDL.
6+
This fix may cause diffs if tables were created with incorrect types.
7+
8+
More context: https://github.com/TobikoData/sqlmesh/pull/5231
9+
"""
10+
11+
import json
12+
13+
from sqlglot import exp
14+
15+
from sqlmesh.core.console import get_console
16+
17+
18+
def migrate(state_sync, **kwargs): # type: ignore
19+
engine_adapter = state_sync.engine_adapter
20+
schema = state_sync.schema
21+
snapshots_table = "_snapshots"
22+
if schema:
23+
snapshots_table = f"{schema}.{snapshots_table}"
24+
25+
warning = (
26+
"IMPORTANT: this migration may have unexpected side-effects for dbt projects.\n\n"
27+
"SQLMesh previously misinterpreted dbt's schema.yml 'data_type' field as actual "
28+
"column types, but dbt only uses these for contracts/validation, not in actual DDL statements. This "
29+
"has been fixed to match dbt's actual behavior. Your existing tables may have been created with "
30+
"incorrect column types. After this migration, run 'sqlmesh diff prod' to check for column "
31+
"type differences, and if any are found, apply a plan to correct the table schemas. "
32+
"For more details, see: https://github.com/TobikoData/sqlmesh/pull/5231."
33+
)
34+
35+
for (snapshot,) in engine_adapter.fetchall(
36+
exp.select("snapshot").from_(snapshots_table), quote_identifiers=True
37+
):
38+
parsed_snapshot = json.loads(snapshot)
39+
node = parsed_snapshot["node"]
40+
41+
if node.get("columns"):
42+
get_console().log_warning(warning)
43+
return

0 commit comments

Comments
 (0)