From f3c288749fad81dec9af4a9c9396d9109a15b0f3 Mon Sep 17 00:00:00 2001 From: George Sittas Date: Wed, 27 Aug 2025 18:30:13 +0300 Subject: [PATCH 1/4] Fix: properly handle {% raw %}...{% endraw %} during rendering --- sqlmesh/core/macros.py | 6 ++++-- sqlmesh/core/renderer.py | 9 ++++++++- tests/core/test_model.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/sqlmesh/core/macros.py b/sqlmesh/core/macros.py index 42a4a8b8dc..6de6e061ff 100644 --- a/sqlmesh/core/macros.py +++ b/sqlmesh/core/macros.py @@ -246,7 +246,9 @@ def send( ) def transform( - self, expression: exp.Expression + self, + expression: exp.Expression, + render_jinja: bool = True, ) -> exp.Expression | t.List[exp.Expression] | None: changed = False @@ -284,7 +286,7 @@ def evaluate_macros( return exp.to_identifier(text, quoted=node.quoted or None) if node.is_string: text = node.this - if has_jinja(text): + if render_jinja and has_jinja(text): changed = True node.set("this", self.jinja_env.from_string(node.this).render()) return node diff --git a/sqlmesh/core/renderer.py b/sqlmesh/core/renderer.py index 8b733d4c55..9a0ce0bc4a 100644 --- a/sqlmesh/core/renderer.py +++ b/sqlmesh/core/renderer.py @@ -230,6 +230,10 @@ def _resolve_table(table: str | exp.Table) -> str: f"Could not render or parse jinja at '{self._path}'.\n{ex}" ) from ex + render_jinja = False + else: + render_jinja = True + macro_evaluator.locals.update(render_kwargs) if variables: @@ -247,7 +251,9 @@ def _resolve_table(table: str | exp.Table) -> str: for expression in expressions: try: - transformed_expressions = ensure_list(macro_evaluator.transform(expression)) + transformed_expressions = ensure_list( + macro_evaluator.transform(expression, render_jinja=render_jinja) + ) except Exception as ex: raise_config_error( f"Failed to resolve macros for\n\n{expression.sql(dialect=self._dialect, pretty=True)}\n\n{ex}\n", @@ -278,6 +284,7 @@ def _resolve_table(table: str | exp.Table) -> str: # MacroEvaluator can resolve columns_to_types calls and provide true schemas. if should_cache and (not self.schema.empty or not macro_evaluator.columns_to_types_called): self._cache = resolved_expressions + return resolved_expressions def update_cache(self, expression: t.Optional[exp.Expression]) -> None: diff --git a/tests/core/test_model.py b/tests/core/test_model.py index eecc3977e7..1d8be6855c 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -11543,3 +11543,18 @@ def test_text_diff_optimize_query(): diff = model1.text_diff(model2) assert diff, "Expected diff to show optimize_query change" assert "+ optimize_query" in diff.lower() + + +def test_raw_jinja_raw_tag(): + expressions = d.parse( + """ + MODEL (name test); + + JINJA_QUERY_BEGIN; + SELECT {% raw %} '{{ foo }}' {% endraw %} AS col; + JINJA_END; + """ + ) + + model = load_sql_based_model(expressions) + assert model.render_query().sql() == "SELECT '{{ foo }}' AS \"col\"" From 9db824e6c9b29a7f74826ce942c35727a0c0dc27 Mon Sep 17 00:00:00 2001 From: George Sittas Date: Wed, 27 Aug 2025 19:21:45 +0300 Subject: [PATCH 2/4] Stop rendering jinja within the macro evaluator --- .../sushi/models/customer_revenue_by_day.sql | 4 ++-- .../models/waiter_as_customer_by_day.sql | 2 +- sqlmesh/core/macros.py | 21 +------------------ sqlmesh/core/renderer.py | 9 +------- tests/core/test_model.py | 10 ++++++--- 5 files changed, 12 insertions(+), 34 deletions(-) diff --git a/examples/sushi/models/customer_revenue_by_day.sql b/examples/sushi/models/customer_revenue_by_day.sql index 3b7f3724cb..aa6a39f396 100644 --- a/examples/sushi/models/customer_revenue_by_day.sql +++ b/examples/sushi/models/customer_revenue_by_day.sql @@ -21,7 +21,7 @@ WITH order_total AS ( LEFT JOIN sushi.items AS i ON oi.item_id = i.id AND oi.event_date = i.event_date WHERE - oi.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND CAST('{{ end_ds }}' as DATE) + oi.event_date BETWEEN @start_ds AND @end_ds GROUP BY oi.order_id, oi.event_date @@ -35,7 +35,7 @@ FROM sushi.orders AS o LEFT JOIN order_total AS ot ON o.id = ot.order_id AND o.event_date = ot.event_date WHERE - o.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND CAST('{{ end_ds }}' as DATE) + o.event_date BETWEEN @start_ds AND @end_ds GROUP BY o.customer_id, o.event_date diff --git a/examples/sushi/models/waiter_as_customer_by_day.sql b/examples/sushi/models/waiter_as_customer_by_day.sql index 7dc12db873..dd9f79b5a3 100644 --- a/examples/sushi/models/waiter_as_customer_by_day.sql +++ b/examples/sushi/models/waiter_as_customer_by_day.sql @@ -27,6 +27,6 @@ SELECT FROM sushi.waiters AS w JOIN sushi.customers as c ON w.waiter_id = c.customer_id JOIN sushi.waiter_names as wn ON w.waiter_id = wn.id -WHERE w.event_date BETWEEN @start_date AND @end_date; +WHERE w.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND @end_date; JINJA_END; diff --git a/sqlmesh/core/macros.py b/sqlmesh/core/macros.py index 6de6e061ff..9e7df5d111 100644 --- a/sqlmesh/core/macros.py +++ b/sqlmesh/core/macros.py @@ -12,7 +12,6 @@ from datetime import datetime, date import sqlglot -from jinja2 import Environment from sqlglot import Generator, exp, parse_one from sqlglot.executor.env import ENV from sqlglot.executor.python import Python @@ -40,7 +39,6 @@ ) from sqlmesh.utils.date import DatetimeRanges, to_datetime, to_date from sqlmesh.utils.errors import MacroEvalError, SQLMeshError -from sqlmesh.utils.jinja import JinjaMacroRegistry, has_jinja from sqlmesh.utils.metaprogramming import ( Executable, SqlValue, @@ -193,7 +191,6 @@ def __init__( self.columns_to_types_called = False self.default_catalog = default_catalog - self._jinja_env: t.Optional[Environment] = None self._schema = schema self._resolve_table = resolve_table self._resolve_tables = resolve_tables @@ -246,9 +243,7 @@ def send( ) def transform( - self, - expression: exp.Expression, - render_jinja: bool = True, + self, expression: exp.Expression ) -> exp.Expression | t.List[exp.Expression] | None: changed = False @@ -284,12 +279,6 @@ def evaluate_macros( if node.this != text: changed = True return exp.to_identifier(text, quoted=node.quoted or None) - if node.is_string: - text = node.this - if render_jinja and has_jinja(text): - changed = True - node.set("this", self.jinja_env.from_string(node.this).render()) - return node if isinstance(node, MacroFunc): changed = True return self.evaluate(node) @@ -438,14 +427,6 @@ def parse_one( """ return sqlglot.maybe_parse(sql, dialect=self.dialect, into=into, **opts) - @property - def jinja_env(self) -> Environment: - if not self._jinja_env: - jinja_env_methods = {**self.locals, **self.env} - del jinja_env_methods["self"] - self._jinja_env = JinjaMacroRegistry().build_environment(**jinja_env_methods) - return self._jinja_env - def columns_to_types(self, model_name: TableName | exp.Column) -> t.Dict[str, exp.DataType]: """Returns the columns-to-types mapping corresponding to the specified model.""" diff --git a/sqlmesh/core/renderer.py b/sqlmesh/core/renderer.py index 9a0ce0bc4a..8b733d4c55 100644 --- a/sqlmesh/core/renderer.py +++ b/sqlmesh/core/renderer.py @@ -230,10 +230,6 @@ def _resolve_table(table: str | exp.Table) -> str: f"Could not render or parse jinja at '{self._path}'.\n{ex}" ) from ex - render_jinja = False - else: - render_jinja = True - macro_evaluator.locals.update(render_kwargs) if variables: @@ -251,9 +247,7 @@ def _resolve_table(table: str | exp.Table) -> str: for expression in expressions: try: - transformed_expressions = ensure_list( - macro_evaluator.transform(expression, render_jinja=render_jinja) - ) + transformed_expressions = ensure_list(macro_evaluator.transform(expression)) except Exception as ex: raise_config_error( f"Failed to resolve macros for\n\n{expression.sql(dialect=self._dialect, pretty=True)}\n\n{ex}\n", @@ -284,7 +278,6 @@ def _resolve_table(table: str | exp.Table) -> str: # MacroEvaluator can resolve columns_to_types calls and provide true schemas. if should_cache and (not self.schema.empty or not macro_evaluator.columns_to_types_called): self._cache = resolved_expressions - return resolved_expressions def update_cache(self, expression: t.Optional[exp.Expression]) -> None: diff --git a/tests/core/test_model.py b/tests/core/test_model.py index 1d8be6855c..9266a56c10 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -2566,11 +2566,15 @@ def test_parse(assert_exp_eq): dialect '', ); + JINJA_QUERY_BEGIN; + SELECT id::INT AS id, ds FROM x - WHERE ds BETWEEN '{{ start_ds }}' AND @end_ds + WHERE ds BETWEEN '{{ start_ds }}' AND @end_ds; + + JINJA_END; """ ) model = load_sql_based_model(expressions, dialect="hive") @@ -2580,8 +2584,8 @@ def test_parse(assert_exp_eq): } assert not model.annotated assert model.dialect == "" - assert isinstance(model.query, exp.Select) - assert isinstance(SqlModel.parse_raw(model.json()).query, exp.Select) + assert isinstance(model.query, d.JinjaQuery) + assert isinstance(SqlModel.parse_raw(model.json()).query, d.JinjaQuery) assert_exp_eq( model.render_query(), """ From cba8afe07f6ce5b59c84d96cec79c165a2355e9b Mon Sep 17 00:00:00 2001 From: George Sittas Date: Wed, 27 Aug 2025 21:25:52 +0300 Subject: [PATCH 3/4] Revert waiter_as_customer_by_day test --- examples/sushi/models/waiter_as_customer_by_day.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sushi/models/waiter_as_customer_by_day.sql b/examples/sushi/models/waiter_as_customer_by_day.sql index dd9f79b5a3..7dc12db873 100644 --- a/examples/sushi/models/waiter_as_customer_by_day.sql +++ b/examples/sushi/models/waiter_as_customer_by_day.sql @@ -27,6 +27,6 @@ SELECT FROM sushi.waiters AS w JOIN sushi.customers as c ON w.waiter_id = c.customer_id JOIN sushi.waiter_names as wn ON w.waiter_id = wn.id -WHERE w.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND @end_date; +WHERE w.event_date BETWEEN @start_date AND @end_date; JINJA_END; From 82edecd20e73c1aba56711892572a79340a6aeee Mon Sep 17 00:00:00 2001 From: George Sittas Date: Wed, 27 Aug 2025 22:28:00 +0300 Subject: [PATCH 4/4] Fix sushi project --- examples/sushi/models/customer_revenue_by_day.sql | 4 ++-- examples/sushi/models/waiter_as_customer_by_day.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/sushi/models/customer_revenue_by_day.sql b/examples/sushi/models/customer_revenue_by_day.sql index aa6a39f396..248af2db8d 100644 --- a/examples/sushi/models/customer_revenue_by_day.sql +++ b/examples/sushi/models/customer_revenue_by_day.sql @@ -21,7 +21,7 @@ WITH order_total AS ( LEFT JOIN sushi.items AS i ON oi.item_id = i.id AND oi.event_date = i.event_date WHERE - oi.event_date BETWEEN @start_ds AND @end_ds + oi.event_date BETWEEN @start_date AND @end_date GROUP BY oi.order_id, oi.event_date @@ -35,7 +35,7 @@ FROM sushi.orders AS o LEFT JOIN order_total AS ot ON o.id = ot.order_id AND o.event_date = ot.event_date WHERE - o.event_date BETWEEN @start_ds AND @end_ds + o.event_date BETWEEN @start_date AND @end_date GROUP BY o.customer_id, o.event_date diff --git a/examples/sushi/models/waiter_as_customer_by_day.sql b/examples/sushi/models/waiter_as_customer_by_day.sql index 7dc12db873..dd9f79b5a3 100644 --- a/examples/sushi/models/waiter_as_customer_by_day.sql +++ b/examples/sushi/models/waiter_as_customer_by_day.sql @@ -27,6 +27,6 @@ SELECT FROM sushi.waiters AS w JOIN sushi.customers as c ON w.waiter_id = c.customer_id JOIN sushi.waiter_names as wn ON w.waiter_id = wn.id -WHERE w.event_date BETWEEN @start_date AND @end_date; +WHERE w.event_date BETWEEN CAST('{{ start_ds }}' as DATE) AND @end_date; JINJA_END;