only sort root dict

Author: eakmanrq

sqlmesh/core/model/common.py

@@ -153,15 +153,15 @@ def _add_variables_to_python_env(
 
     variables = {k: v for k, v in (variables or {}).items() if k in used_variables}
     if variables:
-        python_env[c.SQLMESH_VARS] = Executable.value(variables, use_deterministic_repr=True)
+        python_env[c.SQLMESH_VARS] = Executable.value(variables, sort_root_dict=True)
 
     if blueprint_variables:
         blueprint_variables = {
             k: SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
             for k, v in blueprint_variables.items()
         }
         python_env[c.SQLMESH_BLUEPRINT_VARS] = Executable.value(
-            blueprint_variables, use_deterministic_repr=True
+            blueprint_variables, sort_root_dict=True
         )
 
     return python_env

sqlmesh/migrations/v0085_deterministic_repr.py

@@ -4,6 +4,7 @@
 """
 
 import json
+import logging
 import typing as t
 from dataclasses import dataclass
 
@@ -12,6 +13,9 @@
 from sqlmesh.utils.migration import index_text_type, blob_text_type
 
 
+logger = logging.getLogger(__name__)
+
+
 KEYS_TO_MAKE_DETERMINISTIC = ["__sqlmesh__vars__", "__sqlmesh__blueprint__vars__"]
 
 
@@ -23,25 +27,13 @@ class SqlValue:
     sql: str
 
 
-def _deterministic_repr(obj: t.Any) -> str:
-    """
-    This is a copy of the function from utils.metaprogramming
-    """
-
-    def _normalize_for_repr(o: t.Any) -> t.Any:
-        if isinstance(o, dict):
-            sorted_items = sorted(o.items(), key=lambda x: str(x[0]))
-            return {k: _normalize_for_repr(v) for k, v in sorted_items}
-        if isinstance(o, (list, tuple)):
-            # Recursively normalize nested structures
-            normalized = [_normalize_for_repr(item) for item in o]
-            return type(o)(normalized)
-        return o
-
+def _dict_sort(obj: t.Any) -> str:
     try:
-        return repr(_normalize_for_repr(obj))
+        if isinstance(obj, dict):
+            obj = dict(sorted(obj.items(), key=lambda x: str(x[0])))
     except Exception:
-        return repr(obj)
+        logger.warning("Failed to sort non-recursive dict", exc_info=True)
+    return repr(obj)
 
 
 def migrate(state_sync, **kwargs):  # type: ignore
@@ -92,15 +84,15 @@ def migrate(state_sync, **kwargs):  # type: ignore
                     try:
                         # Try to parse the old payload and re-serialize it deterministically
                         parsed_value = eval(old_payload)
-                        new_payload = _deterministic_repr(parsed_value)
+                        new_payload = _dict_sort(parsed_value)
 
                         # Only update if the representation changed
                         if old_payload != new_payload:
                             executable["payload"] = new_payload
                             migration_needed = True
                     except Exception:
                         # If we still can't eval it, leave it as-is
-                        pass
+                        logger.warning("Exception trying to eval payload", exc_info=True)
 
         new_snapshots.append(
             {

sqlmesh/migrations/v0086_check_deterministic_bug.py

@@ -1,43 +1,15 @@
 import json
-import typing as t
+import logging
+
 from sqlglot import exp
 
 from sqlmesh.core.console import get_console
 
 
+logger = logging.getLogger(__name__)
 KEYS_TO_MAKE_DETERMINISTIC = ["__sqlmesh__vars__", "__sqlmesh__blueprint__vars__"]
 
 
-def would_sorting_be_applied(obj: t.Any) -> bool:
-    """
-    Detects if sorting would be applied to an object based on the
-    deterministic_repr logic.
-
-    Returns True if the object is a dictionary or contains a dictionary
-    at any nesting level (in lists or tuples).
-
-    Args:
-        obj: The object to check
-
-    Returns:
-        bool: True if sorting would be applied, False otherwise
-    """
-
-    def _check_for_dict(o: t.Any) -> bool:
-        if isinstance(o, dict):
-            return True
-        if isinstance(o, (list, tuple)):
-            return any(_check_for_dict(item) for item in o)
-
-        return False
-
-    try:
-        return _check_for_dict(obj)
-    except Exception:
-        # If any error occurs during checking, assume no sorting
-        return False
-
-
 def migrate(state_sync, **kwargs):  # type: ignore
     engine_adapter = state_sync.engine_adapter
     schema = state_sync.schema
@@ -103,8 +75,8 @@ def migrate(state_sync, **kwargs):  # type: ignore
                 ):
                     try:
                         parsed_value = eval(executable["payload"])
-                        if would_sorting_be_applied(parsed_value):
+                        if isinstance(parsed_value, dict):
                             get_console().log_warning(warning)
                             return
                     except Exception:
-                        pass
+                        logger.warning("Exception trying to eval payload", exc_info=True)

sqlmesh/utils/metaprogramming.py

@@ -5,6 +5,7 @@
 import importlib
 import inspect
 import linecache
+import logging
 import os
 import re
 import sys
@@ -23,6 +24,9 @@
 from sqlmesh.utils.errors import SQLMeshError
 from sqlmesh.utils.pydantic import PydanticModel
 
+logger = logging.getLogger(__name__)
+
+
 IGNORE_DECORATORS = {"macro", "model", "signal"}
 SERIALIZABLE_CALLABLES = (type, types.FunctionType)
 LITERALS = (Number, str, bytes, tuple, list, dict, set, bool)
@@ -425,9 +429,9 @@ def is_value(self) -> bool:
 
     @classmethod
     def value(
-        cls, v: t.Any, is_metadata: t.Optional[bool] = None, use_deterministic_repr: bool = False
+        cls, v: t.Any, is_metadata: t.Optional[bool] = None, sort_root_dict: bool = False
     ) -> Executable:
-        payload = _deterministic_repr(v) if use_deterministic_repr else repr(v)
+        payload = _dict_sort(v) if sort_root_dict else repr(v)
         return Executable(payload=payload, kind=ExecutableKind.VALUE, is_metadata=is_metadata)
 
 
@@ -636,36 +640,13 @@ def print_exception(
     out.write(tb)
 
 
-def _deterministic_repr(obj: t.Any) -> str:
-    """Create a deterministic representation by ensuring consistent ordering before repr().
-
-    For dictionaries, ensures consistent key ordering to prevent non-deterministic
-    serialization that affects fingerprinting. Uses Python's native repr() logic
-    for all formatting to handle edge cases properly.
-
-    Note that this function assumes list/tuple order is significant and therefore does not sort them.
-
-    Args:
-        obj: The object to represent as a string.
-
-    Returns:
-        A deterministic string representation of the object.
-    """
-
-    def _normalize_for_repr(o: t.Any) -> t.Any:
-        if isinstance(o, dict):
-            sorted_items = sorted(o.items(), key=lambda x: str(x[0]))
-            return {k: _normalize_for_repr(v) for k, v in sorted_items}
-        if isinstance(o, (list, tuple)):
-            # Recursively normalize nested structures
-            normalized = [_normalize_for_repr(item) for item in o]
-            return type(o)(normalized)
-        return o
-
+def _dict_sort(obj: t.Any) -> str:
     try:
-        return repr(_normalize_for_repr(obj))
+        if isinstance(obj, dict):
+            obj = dict(sorted(obj.items(), key=lambda x: str(x[0])))
     except Exception:
-        return repr(obj)
+        logger.warning("Failed to sort non-recursive dict", exc_info=True)
+    return repr(obj)
 
 
 def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType:

tests/core/test_model.py

@@ -6117,7 +6117,7 @@ def test_named_variable_macros() -> None:
 
     assert model.python_env[c.SQLMESH_VARS] == Executable.value(
         {c.GATEWAY: "in_memory", "test_var_a": "test_value", "overridden_var": "initial_value"},
-        use_deterministic_repr=True,
+        sort_root_dict=True,
     )
     assert (
         model.render_query_or_raise().sql()
@@ -6144,7 +6144,7 @@ def test_variables_in_templates() -> None:
 
     assert model.python_env[c.SQLMESH_VARS] == Executable.value(
         {c.GATEWAY: "in_memory", "test_var_a": "test_value", "overridden_var": "initial_value"},
-        use_deterministic_repr=True,
+        sort_root_dict=True,
     )
     assert (
         model.render_query_or_raise().sql()
@@ -6169,7 +6169,7 @@ def test_variables_in_templates() -> None:
 
     assert model.python_env[c.SQLMESH_VARS] == Executable.value(
         {c.GATEWAY: "in_memory", "test_var_a": "test_value", "overridden_var": "initial_value"},
-        use_deterministic_repr=True,
+        sort_root_dict=True,
     )
     assert (
         model.render_query_or_raise().sql()
@@ -6309,7 +6309,7 @@ def test_variables_migrated_dbt_package_macro():
     )
     assert model.python_env[c.SQLMESH_VARS] == Executable.value(
         {"test_var_a": "test_var_a_value", "__dbt_packages__.test.test_var_b": "test_var_b_value"},
-        use_deterministic_repr=True,
+        sort_root_dict=True,
     )
     assert (
         model.render_query().sql(dialect="bigquery")
@@ -6535,7 +6535,7 @@ def test_unrendered_macros_sql_model(mocker: MockerFixture) -> None:
             "virtual_var": "blb",
             "session_var": "blc",
         },
-        use_deterministic_repr=True,
+        sort_root_dict=True,
     )
 
     assert "location1" in model.physical_properties
@@ -6623,7 +6623,7 @@ def model_with_macros(evaluator, **kwargs):
             "virtual_var": "blb",
             "session_var": "blc",
         },
-        use_deterministic_repr=True,
+        sort_root_dict=True,
     )
     assert python_sql_model.enabled
 
@@ -10583,10 +10583,10 @@ def unimportant_testing_macro(evaluator, *projections):
 
     assert m.python_env.get(c.SQLMESH_VARS) == Executable.value(
         {"selector": "bla", "bla_variable": 1, "baz_variable": 2},
-        use_deterministic_repr=True,
+        sort_root_dict=True,
     )
     assert m.python_env.get(c.SQLMESH_BLUEPRINT_VARS) == Executable.value(
-        {"selector": "baz"}, use_deterministic_repr=True
+        {"selector": "baz"}, sort_root_dict=True
     )