Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sqlmesh/dbt/builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,7 @@ def create_builtin_globals(
builtin_globals.update(
{
"adapter": adapter,
"execute": True,
"execute": isinstance(adapter, RuntimeAdapter),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, we should not do this. execute is set unconditionally to True for a reason. Since we used the rendered query for change detection and categorization we want to follow the execution path during rendering. If we happen to encounter the adapter call we fail and return None which indicates that the query cannot be rendered and categorized.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we used the rendered query for change detection and categorization we want to follow the execution path during rendering. If we happen to encounter the adapter call we fail and return None which indicates that the query cannot be rendered and categorized.

I see was wondering about this, thanks that makes sense. and I guess now that we can fail and still proceed later to runtime this can't lead to any issues

"load_relation": adapter.load_relation,
"store_result": sql_execution.store_result,
"load_result": sql_execution.load_result,
Expand Down
9 changes: 5 additions & 4 deletions tests/core/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -2068,21 +2068,22 @@ def test_dbt_is_incremental_table_is_missing(sushi_test_dbt_context: Context):
assert context.engine_adapter.table_exists(snapshot.table_name())


@pytest.mark.xdist_group("dbt_manifest")
def test_model_attr(sushi_test_dbt_context: Context, assert_exp_eq):
context = sushi_test_dbt_context
model = context.get_model("sushi.top_waiters")
assert_exp_eq(
model.render_query(),
"""
SELECT
CAST("waiter_id" AS INT) AS "waiter_id",
CAST("revenue" AS DOUBLE) AS "revenue",
CAST("waiter_revenue_by_day_v2"."waiter_id" AS INT) AS "waiter_id",
CAST("waiter_revenue_by_day_v2"."revenue" AS DOUBLE) AS "revenue",
3 AS "model_columns"
FROM "memory"."sushi"."waiter_revenue_by_day_v2" AS "waiter_revenue_by_day_v2"
WHERE
"ds" = (
"waiter_revenue_by_day_v2"."ds" = (
SELECT
MAX("ds")
MAX("waiter_revenue_by_day_v2"."ds") AS "_col_0"
FROM "memory"."sushi"."waiter_revenue_by_day_v2" AS "waiter_revenue_by_day_v2"
)
ORDER BY
Expand Down
49 changes: 47 additions & 2 deletions tests/dbt/test_transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from pathlib import Path
from unittest.mock import patch

from sqlmesh.dbt.adapter import ParsetimeAdapter, RuntimeAdapter
from sqlmesh.dbt.util import DBT_VERSION

import pytest
Expand Down Expand Up @@ -43,7 +44,7 @@
OnAdditiveChange,
)
from sqlmesh.core.state_sync.db.snapshot import _snapshot_to_json
from sqlmesh.dbt.builtin import _relation_info_to_relation, Config
from sqlmesh.dbt.builtin import _relation_info_to_relation, Config, create_builtin_globals
from sqlmesh.dbt.common import Dependencies
from sqlmesh.dbt.column import (
ColumnConfig,
Expand All @@ -64,7 +65,7 @@
)
from sqlmesh.dbt.test import TestConfig
from sqlmesh.utils.errors import ConfigError, MacroEvalError, SQLMeshError
from sqlmesh.utils.jinja import MacroReference
from sqlmesh.utils.jinja import JinjaMacroRegistry, MacroReference

pytestmark = [pytest.mark.dbt, pytest.mark.slow]

Expand Down Expand Up @@ -2352,3 +2353,47 @@ def test_dynamic_var_names_in_macro(sushi_test_project: Project):
)
converted_model = model_config.to_sqlmesh(context)
assert "dynamic_test_var" in converted_model.jinja_macros.global_objs["vars"] # type: ignore


@pytest.mark.xdist_group("dbt_manifest")
def test_execute_variable_parse_vs_runtime(sushi_test_dbt_context: Context):
execute_model = sushi_test_dbt_context.get_model('"memory"."sushi"."execute_test_model"')
parse_time_query = execute_model.render_query()
if parse_time_query:
parse_sql = parse_time_query.sql()
# should contain parse-time placeholders
assert "parse_time_placeholder" in parse_sql or "parse_time_context" in parse_sql

runtime_query = execute_model.render_query_or_raise(
engine_adapter=sushi_test_dbt_context.engine_adapter
)
runtime_sql = runtime_query.sql()
# At runtime, the macro should have executed the query and returned the result
assert "1" in runtime_sql or "runtime_context" in runtime_sql


@pytest.mark.xdist_group("dbt_manifest")
def test_execute_globals(mocker: MockerFixture):
# No engine adapter shoulde create parse adapter
parse_time_globals = create_builtin_globals(
jinja_macros=JinjaMacroRegistry(),
jinja_globals={},
engine_adapter=None,
)

assert parse_time_globals["execute"] is False
assert parse_time_globals["flags"].WHICH == "parse"
assert isinstance(parse_time_globals["adapter"], ParsetimeAdapter)

mock_engine_adapter = mocker.Mock()

# Runtime globals should have execute=True and RuntimeAdapter
runtime_globals = create_builtin_globals(
jinja_macros=JinjaMacroRegistry(),
jinja_globals={},
engine_adapter=mock_engine_adapter,
)

assert runtime_globals["execute"] is True
assert runtime_globals["flags"].WHICH == "run"
assert isinstance(runtime_globals["adapter"], RuntimeAdapter)
9 changes: 9 additions & 0 deletions tests/fixtures/dbt/sushi_test/macros/execute_test.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{% macro runtime_sql() %}
{% if execute %}
{% set result = run_query("SELECT 1 as test_col") %}
{% set test_value = result.columns[0][0] %}
{{ return(test_value) }}
{% else %}
{{ return("parse_time_placeholder") }}
{% endif %}
{% endmacro %}
2 changes: 2 additions & 0 deletions tests/fixtures/dbt/sushi_test/models/execute_test_model.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
SELECT
'{{ runtime_sql() }}' as runtime_result
Loading