From 992ca0c058a929e2f12f3c76c0f1443c6dd4346e Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Wed, 3 Sep 2025 22:47:33 +0000 Subject: [PATCH 1/7] Chore: improve test stability and address some warnings --- pyproject.toml | 6 +++++- sqlmesh/core/test/context.py | 2 ++ sqlmesh/dbt/test.py | 4 ++++ tests/conftest.py | 4 ---- tests/dbt/test_integration.py | 4 ++++ tests/engines/spark/test_db_api.py | 5 +---- tests/integrations/github/cicd/test_github_commands.py | 2 ++ 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d3886562d6..7550ffdd35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -265,7 +265,11 @@ markers = [ "snowflake: test for Snowflake", "spark: test for Spark", "trino: test for Trino (all connectors)", - "risingwave: test for Risingwave" + "risingwave: test for Risingwave", + + # Other + "set_default_connection", + "registry_isolation" ] addopts = "-n 0 --dist=loadgroup" asyncio_default_fixture_loop_scope = "session" diff --git a/sqlmesh/core/test/context.py b/sqlmesh/core/test/context.py index 5ad9673ca8..a326c3c1b3 100644 --- a/sqlmesh/core/test/context.py +++ b/sqlmesh/core/test/context.py @@ -18,6 +18,8 @@ class TestExecutionContext(ExecutionContext): models: All upstream models to use for expansion and mapping of physical locations. """ + __test__ = False # prevent pytest trying to collect this as a test class + def __init__( self, engine_adapter: EngineAdapter, diff --git a/sqlmesh/dbt/test.py b/sqlmesh/dbt/test.py index 035c62acda..4e802c5120 100644 --- a/sqlmesh/dbt/test.py +++ b/sqlmesh/dbt/test.py @@ -61,6 +61,10 @@ class TestConfig(GeneralConfig): error_if: Conditional expression (default "!=0") to detect if error condition met (Not supported). """ + __test__ = ( + False # prevent pytest trying to collect this as a test class when it's imported in a test + ) + # SQLMesh fields path: Path = Path() name: str diff --git a/tests/conftest.py b/tests/conftest.py index b6523f72a4..8b1315d46a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -222,10 +222,6 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): # note: the hook always has to yield outcome = yield - # we only care about tests that used the tmp_path fixture - if "tmp_path" not in getattr(item, "fixturenames", []): - return - result: pytest.TestReport = outcome.get_result() if result.when != "teardown": diff --git a/tests/dbt/test_integration.py b/tests/dbt/test_integration.py index ee8c486ab2..5a944d55d4 100644 --- a/tests/dbt/test_integration.py +++ b/tests/dbt/test_integration.py @@ -27,6 +27,8 @@ class TestType(str, Enum): + __test__ = False # prevent pytest trying to collect this as a test class + DBT_RUNTIME = "dbt_runtime" DBT_ADAPTER = "dbt_adapter" SQLMESH = "sqlmesh" @@ -53,6 +55,8 @@ def is_sqlmesh_runtime(self) -> bool: class TestStrategy(str, Enum): + __test__ = False # prevent pytest trying to collect this as a test class + CHECK = "check" TIMESTAMP = "timestamp" diff --git a/tests/engines/spark/test_db_api.py b/tests/engines/spark/test_db_api.py index 8bbfe7e9ab..d6af3eee7b 100644 --- a/tests/engines/spark/test_db_api.py +++ b/tests/engines/spark/test_db_api.py @@ -4,10 +4,7 @@ from sqlmesh.engines.spark.db_api import errors from sqlmesh.engines.spark.db_api import spark_session as spark_session_db -pytestmark = [ - pytest.mark.slow, - pytest.mark.spark_pyspark, -] +pytestmark = [pytest.mark.slow, pytest.mark.spark] def test_spark_session_cursor(spark_session: SparkSession): diff --git a/tests/integrations/github/cicd/test_github_commands.py b/tests/integrations/github/cicd/test_github_commands.py index 296fea5938..01e4c9af31 100644 --- a/tests/integrations/github/cicd/test_github_commands.py +++ b/tests/integrations/github/cicd/test_github_commands.py @@ -5,6 +5,8 @@ from unittest import TestCase, mock from unittest.result import TestResult +TestResult.__test__ = False # prevent pytest trying to collect this as a test class + import pytest from pytest_mock.plugin import MockerFixture From a7dac017c0c016343c353f9d7580a61be28c2bbb Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Wed, 3 Sep 2025 22:49:46 +0000 Subject: [PATCH 2/7] enable cloud tests --- .circleci/continue_config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index e21f3d869b..40d7c92974 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -310,10 +310,10 @@ workflows: - athena - fabric - gcp-postgres - filters: - branches: - only: - - main + #filters: + # branches: + # only: + # - main - ui_style - ui_test - vscode_test From 113825b1ace10fd43f8d5972e1165ecdf495a149 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Wed, 3 Sep 2025 23:12:12 +0000 Subject: [PATCH 3/7] Actually run spark tests --- .circleci/install-prerequisites.sh | 5 +++++ Makefile | 2 +- pyproject.toml | 1 + tests/engines/spark/test_db_api.py | 5 ++++- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.circleci/install-prerequisites.sh b/.circleci/install-prerequisites.sh index cbd8491535..446221dba6 100755 --- a/.circleci/install-prerequisites.sh +++ b/.circleci/install-prerequisites.sh @@ -34,4 +34,9 @@ echo "Installing OS-level dependencies: $ALL_DEPENDENCIES" sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES +if [ "$ENGINE" == "spark" ]; then + echo "Using Java version for spark:" + java -version +fi + echo "All done" \ No newline at end of file diff --git a/Makefile b/Makefile index 668769e2ef..d347d0ce21 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,7 @@ postgres-test: engine-postgres-up pytest -n auto -m "postgres" --retries 3 --junitxml=test-results/junit-postgres.xml spark-test: engine-spark-up - pytest -n auto -m "spark" --retries 3 --junitxml=test-results/junit-spark.xml + pytest -n auto -m "spark" --retries 3 --junitxml=test-results/junit-spark.xml && pytest -n auto -m "pyspark" --retries 3 --junitxml=test-results/junit-pyspark.xml trino-test: engine-trino-up pytest -n auto -m "trino" --retries 3 --junitxml=test-results/junit-trino.xml diff --git a/pyproject.toml b/pyproject.toml index 7550ffdd35..bde66b20ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -264,6 +264,7 @@ markers = [ "redshift: test for Redshift", "snowflake: test for Snowflake", "spark: test for Spark", + "pyspark: test for PySpark that need to run separately from the other spark tests", "trino: test for Trino (all connectors)", "risingwave: test for Risingwave", diff --git a/tests/engines/spark/test_db_api.py b/tests/engines/spark/test_db_api.py index d6af3eee7b..eab7a0c223 100644 --- a/tests/engines/spark/test_db_api.py +++ b/tests/engines/spark/test_db_api.py @@ -4,7 +4,10 @@ from sqlmesh.engines.spark.db_api import errors from sqlmesh.engines.spark.db_api import spark_session as spark_session_db -pytestmark = [pytest.mark.slow, pytest.mark.spark] +# note: this is deliberately not marked with 'spark' so that it +# can run separately from the spark integration tests. +# running them at the same time mutates some global state in the SparkSession which breaks these tests +pytestmark = [pytest.mark.slow, pytest.mark.pyspark] def test_spark_session_cursor(spark_session: SparkSession): From ed8f91eb0c122b819767cde31c2d5ef23a8bcbdb Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Thu, 4 Sep 2025 05:04:40 +0000 Subject: [PATCH 4/7] shotgun debugging --- tests/conftest.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8b1315d46a..7441599f06 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -225,20 +225,16 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): result: pytest.TestReport = outcome.get_result() if result.when != "teardown": - return + return result # If we specifically failed with a StashKey error in teardown, mark the test as passed - if result.failed: - exception = call.excinfo - if ( - exception - and isinstance(exception.value, KeyError) - and "_pytest.stash.StashKey" in repr(exception) - ): - result.outcome = "passed" - item.add_report_section( - "teardown", "stderr", f"Ignored tmp_path teardown error: {exception}" - ) + if (exception := call.excinfo) and "_pytest.stash.StashKey" in repr(exception): + call.excinfo = None + result.outcome = "passed" + item.add_report_section( + "teardown", "stderr", f"Ignored tmp_path teardown error: {exception}" + ) + return result def pytest_configure(config: pytest.Config): From 053d2ceacab8eaf3f752bc940056442decf82fc3 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Thu, 4 Sep 2025 05:20:21 +0000 Subject: [PATCH 5/7] re-enable databricks --- .circleci/manage-test-db.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/manage-test-db.sh b/.circleci/manage-test-db.sh index f79072f335..f90b567ce8 100755 --- a/.circleci/manage-test-db.sh +++ b/.circleci/manage-test-db.sh @@ -51,7 +51,9 @@ databricks_init() { # Note: the cluster doesnt need to be running to create / drop catalogs, but it does need to be running to run the integration tests echo "Ensuring cluster is running" - databricks clusters start $CLUSTER_ID + # the || true is to prevent the following error from causing an abort: + # > Error: is in unexpected state Running. + databricks clusters start $CLUSTER_ID || true } databricks_up() { From 6dc4a83f2d8fcf157786a1b1fcbcd2d969dc851f Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Thu, 4 Sep 2025 06:29:07 +0000 Subject: [PATCH 6/7] Switch to pytest-rerunfailures to see if that helps --- Makefile | 34 +++++++++++------------ pyproject.toml | 4 +-- tests/conftest.py | 70 ----------------------------------------------- 3 files changed, 19 insertions(+), 89 deletions(-) diff --git a/Makefile b/Makefile index d347d0ce21..96305c4bfb 100644 --- a/Makefile +++ b/Makefile @@ -138,7 +138,7 @@ dbt-test: pytest -n auto -m "dbt and not cicdonly" dbt-fast-test: - pytest -n auto -m "dbt and fast" --retries 3 + pytest -n auto -m "dbt and fast" --reruns 3 github-test: pytest -n auto -m "github" @@ -173,58 +173,58 @@ engine-%-down: ################## clickhouse-test: engine-clickhouse-up - pytest -n auto -m "clickhouse" --retries 3 --junitxml=test-results/junit-clickhouse.xml + pytest -n auto -m "clickhouse" --reruns 3 --junitxml=test-results/junit-clickhouse.xml duckdb-test: engine-duckdb-install - pytest -n auto -m "duckdb" --retries 3 --junitxml=test-results/junit-duckdb.xml + pytest -n auto -m "duckdb" --reruns 3 --junitxml=test-results/junit-duckdb.xml mssql-test: engine-mssql-up - pytest -n auto -m "mssql" --retries 3 --junitxml=test-results/junit-mssql.xml + pytest -n auto -m "mssql" --reruns 3 --junitxml=test-results/junit-mssql.xml mysql-test: engine-mysql-up - pytest -n auto -m "mysql" --retries 3 --junitxml=test-results/junit-mysql.xml + pytest -n auto -m "mysql" --reruns 3 --junitxml=test-results/junit-mysql.xml postgres-test: engine-postgres-up - pytest -n auto -m "postgres" --retries 3 --junitxml=test-results/junit-postgres.xml + pytest -n auto -m "postgres" --reruns 3 --junitxml=test-results/junit-postgres.xml spark-test: engine-spark-up - pytest -n auto -m "spark" --retries 3 --junitxml=test-results/junit-spark.xml && pytest -n auto -m "pyspark" --retries 3 --junitxml=test-results/junit-pyspark.xml + pytest -n auto -m "spark" --reruns 3 --junitxml=test-results/junit-spark.xml && pytest -n auto -m "pyspark" --reruns 3 --junitxml=test-results/junit-pyspark.xml trino-test: engine-trino-up - pytest -n auto -m "trino" --retries 3 --junitxml=test-results/junit-trino.xml + pytest -n auto -m "trino" --reruns 3 --junitxml=test-results/junit-trino.xml risingwave-test: engine-risingwave-up - pytest -n auto -m "risingwave" --retries 3 --junitxml=test-results/junit-risingwave.xml + pytest -n auto -m "risingwave" --reruns 3 --junitxml=test-results/junit-risingwave.xml ################# # Cloud Engines # ################# snowflake-test: guard-SNOWFLAKE_ACCOUNT guard-SNOWFLAKE_WAREHOUSE guard-SNOWFLAKE_DATABASE guard-SNOWFLAKE_USER guard-SNOWFLAKE_PASSWORD engine-snowflake-install - pytest -n auto -m "snowflake" --retries 3 --junitxml=test-results/junit-snowflake.xml + pytest -n auto -m "snowflake" --reruns 3 --junitxml=test-results/junit-snowflake.xml bigquery-test: guard-BIGQUERY_KEYFILE engine-bigquery-install $(PIP) install -e ".[bigframes]" - pytest -n auto -m "bigquery" --retries 3 --junitxml=test-results/junit-bigquery.xml + pytest -n auto -m "bigquery" --reruns 3 --junitxml=test-results/junit-bigquery.xml databricks-test: guard-DATABRICKS_CATALOG guard-DATABRICKS_SERVER_HOSTNAME guard-DATABRICKS_HTTP_PATH guard-DATABRICKS_ACCESS_TOKEN guard-DATABRICKS_CONNECT_VERSION engine-databricks-install $(PIP) install 'databricks-connect==${DATABRICKS_CONNECT_VERSION}' - pytest -n auto -m "databricks" --retries 3 --junitxml=test-results/junit-databricks.xml + pytest -n auto -m "databricks" --reruns 3 --junitxml=test-results/junit-databricks.xml redshift-test: guard-REDSHIFT_HOST guard-REDSHIFT_USER guard-REDSHIFT_PASSWORD guard-REDSHIFT_DATABASE engine-redshift-install - pytest -n auto -m "redshift" --retries 3 --junitxml=test-results/junit-redshift.xml + pytest -n auto -m "redshift" --reruns 3 --junitxml=test-results/junit-redshift.xml clickhouse-cloud-test: guard-CLICKHOUSE_CLOUD_HOST guard-CLICKHOUSE_CLOUD_USERNAME guard-CLICKHOUSE_CLOUD_PASSWORD engine-clickhouse-install - pytest -n 1 -m "clickhouse_cloud" --retries 3 --junitxml=test-results/junit-clickhouse-cloud.xml + pytest -n 1 -m "clickhouse_cloud" --reruns 3 --junitxml=test-results/junit-clickhouse-cloud.xml athena-test: guard-AWS_ACCESS_KEY_ID guard-AWS_SECRET_ACCESS_KEY guard-ATHENA_S3_WAREHOUSE_LOCATION engine-athena-install - pytest -n auto -m "athena" --retries 3 --junitxml=test-results/junit-athena.xml + pytest -n auto -m "athena" --reruns 3 --junitxml=test-results/junit-athena.xml fabric-test: guard-FABRIC_HOST guard-FABRIC_CLIENT_ID guard-FABRIC_CLIENT_SECRET guard-FABRIC_DATABASE engine-fabric-install - pytest -n auto -m "fabric" --retries 3 --junitxml=test-results/junit-fabric.xml + pytest -n auto -m "fabric" --reruns 3 --junitxml=test-results/junit-fabric.xml gcp-postgres-test: guard-GCP_POSTGRES_INSTANCE_CONNECTION_STRING guard-GCP_POSTGRES_USER guard-GCP_POSTGRES_PASSWORD guard-GCP_POSTGRES_KEYFILE_JSON engine-gcppostgres-install - pytest -n auto -m "gcp_postgres" --retries 3 --junitxml=test-results/junit-gcp-postgres.xml + pytest -n auto -m "gcp_postgres" --reruns 3 --junitxml=test-results/junit-gcp-postgres.xml vscode_settings: mkdir -p .vscode diff --git a/pyproject.toml b/pyproject.toml index bde66b20ab..9ee91ef1a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ dev = [ "pytest", "pytest-asyncio", "pytest-mock", - "pytest-retry", + "pytest-rerunfailures", "pytest-xdist", "pytz", "redshift_connector", @@ -280,7 +280,7 @@ log_cli_level = "INFO" filterwarnings = [ "ignore:The localize method is no longer necessary, as this time zone supports the fold attribute" ] -retry_delay = 10 +reruns_delay = 10 [tool.ruff] line-length = 100 diff --git a/tests/conftest.py b/tests/conftest.py index 7441599f06..e5bbc4f425 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -212,76 +212,6 @@ def pytest_collection_modifyitems(items, *args, **kwargs): item.add_marker("fast") -@pytest.hookimpl(hookwrapper=True, tryfirst=True) -def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): - # The tmp_path fixture frequently throws errors like: - # - KeyError: <_pytest.stash.StashKey object at 0x79ba385fe1a0> - # in its teardown. This causes pytest to mark the test as failed even though we have zero control over this behaviour. - # So we log/swallow that particular error here rather than raising it - - # note: the hook always has to yield - outcome = yield - - result: pytest.TestReport = outcome.get_result() - - if result.when != "teardown": - return result - - # If we specifically failed with a StashKey error in teardown, mark the test as passed - if (exception := call.excinfo) and "_pytest.stash.StashKey" in repr(exception): - call.excinfo = None - result.outcome = "passed" - item.add_report_section( - "teardown", "stderr", f"Ignored tmp_path teardown error: {exception}" - ) - return result - - -def pytest_configure(config: pytest.Config): - # we need to adjust the hook order if pytest-retry is present because it: - # - also declares a `pytest_runtest_makereport` with `hookwrapper=True, tryfirst=True` - # - this supersedes our one because pytest always loads plugins first and they take precedence over user code - # - # but, we need our one to run first because it's capturing and ignoring certain errors that cause pytest-retry to fail - # and not retry. so we need to adjust the order the hooks are called which pytest does NOT make easy. - # - # we can't just unload the pytest-retry plugin, load our hook and reload the pytest-retry plugin either. - # this causes an error: - # > Hook 'pytest_set_excluded_exceptions' is already registered within namespace - # because unregister() apparently doesnt unregister plugins cleanly in such a way they can be re-registered - # - # so what we end up doing below is a small monkey-patch to adjust the call order of the hooks - pm = config.pluginmanager - - from pluggy._hooks import HookCaller - - hook_caller: HookCaller = pm.hook.pytest_runtest_makereport - hook_impls = hook_caller.get_hookimpls() - - # find the index of our one - our_makereport_idx = next( - (i for i, v in enumerate(hook_impls) if v.plugin_name.endswith("tests/conftest.py")), None - ) - - # find the index of the pytest-retry one - pytest_retry_makereport_idx = next( - (i for i, v in enumerate(hook_impls) if v.plugin_name == "pytest-retry"), None - ) - - if ( - pytest_retry_makereport_idx is not None - and our_makereport_idx is not None - and our_makereport_idx > pytest_retry_makereport_idx - ): - our_makereport_hook = hook_impls.pop(our_makereport_idx) - - # inject our one to run before the pytest-retry one - hook_impls.insert(pytest_retry_makereport_idx, our_makereport_hook) - - # HookCaller doesnt have a setter method for this. - hook_caller._hookimpls = hook_impls # type: ignore - - # Ignore all local config files @pytest.fixture(scope="session", autouse=True) def ignore_local_config_files(): From 1a06236fd10f6635cfd630c269f0926a8a34d0b1 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Thu, 4 Sep 2025 17:21:39 +0000 Subject: [PATCH 7/7] Revert "enable cloud tests" This reverts commit a7dac017c0c016343c353f9d7580a61be28c2bbb. --- .circleci/continue_config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 40d7c92974..e21f3d869b 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -310,10 +310,10 @@ workflows: - athena - fabric - gcp-postgres - #filters: - # branches: - # only: - # - main + filters: + branches: + only: + - main - ui_style - ui_test - vscode_test