Fix: always make dataframes hashable when comparing them in unit tests (#3354)

Author: georgesittas

sqlmesh/core/test/definition.py

@@ -237,10 +237,11 @@ def _to_hashable(x: t.Any) -> t.Any:
                 return tuple((k, _to_hashable(v)) for k, v in x.items())
             return str(x) if not isinstance(x, t.Hashable) else x
 
+        actual = actual.apply(lambda col: col.map(_to_hashable))
+        expected = expected.apply(lambda col: col.map(_to_hashable))
+
         if sort:
-            actual = actual.apply(lambda col: col.map(_to_hashable))
             actual = actual.sort_values(by=actual.columns.to_list()).reset_index(drop=True)
-            expected = expected.apply(lambda col: col.map(_to_hashable))
             expected = expected.sort_values(by=expected.columns.to_list()).reset_index(drop=True)
 
         try:

sqlmesh/core/test/result.py

@@ -4,6 +4,12 @@
 import typing as t
 import unittest
 
+if t.TYPE_CHECKING:
+    ErrorType = t.Union[
+        t.Tuple[type[BaseException], BaseException, types.TracebackType],
+        t.Tuple[None, None, None],
+    ]
+
 
 class ModelTextTestResult(unittest.TextTestResult):
     successes: t.List[unittest.TestCase]
@@ -12,13 +18,28 @@ def __init__(self, *args: t.Any, **kwargs: t.Any):
         super().__init__(*args, **kwargs)
         self.successes = []
 
-    def addFailure(
+    def addSubTest(
         self,
         test: unittest.TestCase,
-        err: (
-            tuple[type[BaseException], BaseException, types.TracebackType] | tuple[None, None, None]
-        ),
+        subtest: unittest.TestCase,
+        err: t.Optional[ErrorType],
     ) -> None:
+        """Called at the end of a subtest.
+
+        The traceback is suppressed because it is redundant and not useful.
+
+        Args:
+            test: The test case.
+            subtest: The subtest instance.
+            err: A tuple of the form returned by sys.exc_info(), i.e., (type, value, traceback).
+        """
+        if err:
+            exctype, value, tb = err
+            err = (exctype, value, None)  # type: ignore
+
+        super().addSubTest(test, subtest, err)
+
+    def addFailure(self, test: unittest.TestCase, err: ErrorType) -> None:
         """Called when the test case test signals a failure.
 
         The traceback is suppressed because it is redundant and not useful.

tests/core/test_test.py

@@ -345,6 +345,48 @@ def test_row_order(sushi_context: Context, full_model_without_ctes: SqlModel) ->
         ),
     )
 
+    model_sql = """
+SELECT
+    ARRAY_AGG(DISTINCT id_contact_b ORDER BY id_contact_b) AS aggregated_duplicates
+FROM
+    source
+GROUP BY
+    id_contact_a
+ORDER BY
+    id_contact_a
+    """
+
+    _check_successful_or_raise(
+        _create_test(
+            body=load_yaml(
+                """
+test_array_order:
+  model: test
+  inputs:
+    source:
+    - id_contact_a: a
+      id_contact_b: b
+    - id_contact_a: a
+      id_contact_b: c
+  outputs:
+    query:
+    - aggregated_duplicates:
+      - c
+      - b
+                """
+            ),
+            test_name="test_array_order",
+            model=_create_model(model_sql),
+            context=Context(config=Config(model_defaults=ModelDefaultsConfig(dialect="duckdb"))),
+        ).run(),
+        expected_msg=(
+            """AssertionError: Data mismatch (exp: expected, act: actual)\n\n"""
+            "  aggregated_duplicates        \n"
+            "                    exp     act\n"
+            "0                (c, b)  (b, c)\n"
+        ),
+    )
+
 
 @pytest.mark.parametrize(
     "waiter_names_input",