chore: reorganize test marks (#1933)

* chore: reorganize test marks

* engine integration test fixes

* engine integration test fixes

* change unit/integration to fast/slow

* fix engine marks

Author: eakmanrq

.circleci/continue_config.yml

@@ -37,7 +37,7 @@ commands:
             - run: circleci-agent step halt
 
 jobs:
-  style_and_unit_tests:
+  style_and_slow_tests:
     parameters:
       python_version:
         type: string
@@ -57,13 +57,13 @@ jobs:
           name: Run linters and code style checks
           command: make py-style
       - run:
-          name: Run unit tests
-          command: make unit-test
+          name: Run slow tests
+          command: make slow-test
       - run:
           name: Run doc tests
           command: make doc-test
 
-  style_and_unit_tests_pydantic_v1:
+  style_and_slow_tests_pydantic_v1:
     docker:
       - image: cimg/python:3.10
     resource_class: small
@@ -83,8 +83,8 @@ jobs:
           name: Run linters and code style checks
           command: make py-style
       - run:
-          name: Run unit tests
-          command: make unit-test
+          name: Run slow tests
+          command: make slow-test
 
   ui_style:
     docker:
@@ -132,21 +132,7 @@ jobs:
           name: Run tests
           command: npm --prefix web/client run test
 
-  core_integration_tests:
-    docker:
-      - image: cimg/python:3.7
-    resource_class: small
-    steps:
-      - halt_unless_core
-      - checkout
-      - run:
-          name: Install dependencies
-          command: make install-dev
-      - run:
-          name: Run Core integration tests
-          command: make core-it-test
-
-  airflow_integration_tests:
+  airflow_docker_tests:
     machine:
       image: ubuntu-2204:2022.10.2
       docker_layer_caching: true
@@ -160,8 +146,8 @@ jobs:
           name: Install ruamel.yaml
           command: pip3 install ruamel.yaml==0.16.0
       - run:
-          name: Run Airflow integration tests
-          command: make airflow-it-test-docker-with-env
+          name: Run Airflow slow tests
+          command: make airflow-docker-test-with-env
           no_output_timeout: 15m
       - run:
           name: Collect Airflow logs
@@ -173,7 +159,7 @@ jobs:
       - store_artifacts:
           path: /tmp/airflow_logs
 
-  engine_adapter_integration_local_only_tests:
+  engine_adapter_docker_tests:
     machine:
       image: ubuntu-2204:2022.10.2
       docker_layer_caching: true
@@ -185,39 +171,38 @@ jobs:
           command: sudo apt-get update && sudo apt-get install libpq-dev
       - run:
           name: Install dependencies
-          command: make install-engine-integration
+          command: make install-engine-test
       - run:
-          name: Bring up MySQL, Postgres, and MSSQL
-          command: make core_engine_it_test_docker
+          name: Bring up Dockerized Engines
+          command: make engine-up
       - run:
           name: Make sure DBs are ready
           command: sleep 60
       - run:
           name: Run tests
-          command: make core_engine_it_test_local_only
+          command: make engine-docker-test
           no_output_timeout: 15m
 
 workflows:
   main_pr:
     jobs:
-      - style_and_unit_tests:
+      - style_and_slow_tests:
           matrix:
             parameters:
               python_version:
                 ["3.7", "3.8", "3.9", "3.10", "3.11"]
-      - style_and_unit_tests_pydantic_v1
-      - core_integration_tests
-      - airflow_integration_tests:
+      - style_and_slow_tests_pydantic_v1
+      - airflow_docker_tests:
           requires:
-            - style_and_unit_tests
+            - style_and_slow_tests
           filters:
             branches:
               only:
                 - main
-      - engine_adapter_integration_local_only_tests:
-          context: engine_adapter_integration
+      - engine_adapter_docker_tests:
+          context: engine_adapter_slow
           requires:
-            - style_and_unit_tests
+            - style_and_slow_tests
           filters:
             branches:
               only:

Makefile

@@ -3,7 +3,7 @@
 install-dev:
 	pip3 install -e ".[dev,web,slack]"
 
-install-engine-integration:
+install-engine-test:
 	pip3 install -e ".[dev,web,slack,mysql,postgres,databricks,redshift,bigquery,snowflake,trino,mssql]"
 
 install-pre-commit:
@@ -18,32 +18,9 @@ py-style:
 ui-style:
 	SKIP=autoflake,isort,black,mypy pre-commit run --all-files
 
-unit-test:
-	pytest -m "not integration"
-
 doc-test:
 	PYTEST_PLUGINS=tests.common_fixtures pytest --doctest-modules sqlmesh/core sqlmesh/utils
 
-core-it-test:
-	pytest -m "core_integration"
-
-core_engine_it_test:
-	pytest -m "engine_integration"
-
-core_engine_it_test_local_only:
-	pytest -m "engine_integration_local"
-
-core_engine_it_test_docker:
-	docker-compose -f ./tests/core/engine_adapter/docker-compose.yaml up -d
-
-engine_it_test: core_engine_it_test_docker core_engine_it_test
-
-it-test: core-it-test airflow-it-test-with-env
-
-it-test-docker: core-it-test airflow-it-test-docker-with-env
-
-test: unit-test doc-test it-test
-
 package:
 	pip3 install wheel && python3 setup.py sdist bdist_wheel
 
@@ -71,17 +48,6 @@ airflow-psql:
 airflow-spark-sql:
 	make -C ./examples/airflow spark-sql
 
-airflow-it-test:
-	export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@localhost/airflow && \
-		pytest -m "airflow_integration"
-
-airflow-it-test-docker:
-	make -C ./examples/airflow it-test-docker
-
-airflow-it-test-with-env: develop airflow-clean airflow-init airflow-run airflow-it-test airflow-stop
-
-airflow-it-test-docker-with-env: develop airflow-clean airflow-init airflow-run airflow-it-test-docker airflow-stop
-
 docs-serve:
 	mkdocs serve
 
@@ -106,4 +72,96 @@ clean-build:
 dev-publish: ui-build clean-build publish
 
 jupyter-example:
-	jupyter lab tests/integrations/jupyter/example_outputs.ipynb
+	jupyter lab tests/slows/jupyter/example_outputs.ipynb
+
+engine-up:
+	docker-compose -f ./tests/core/engine_adapter/docker-compose.yaml up -d
+
+engine-down:
+	docker-compose -f ./tests/core/engine_adapter/docker-compose.yaml down
+
+fast-test:
+	pytest -m "fast"
+
+slow-test:
+	pytest -m "fast or slow"
+
+core-fast-test:
+	pytest -m "fast and not web and not github and not dbt and not airflow and not jupyter"
+
+core-slow-test:
+	pytest -m "(fast or slow) and not web and not github and not dbt and not airflow and not jupyter"
+
+airflow-fast-test:
+	pytest -m "fast and airflow"
+
+airflow-test:
+	pytest -m "(fast or slow) and airflow"
+
+airflow-local-test:
+	export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@localhost/airflow && \
+		pytest -m "docker and airflow"
+
+airflow-docker-test:
+	make -C ./examples/airflow docker-test
+
+airflow-local-test-with-env: develop airflow-clean airflow-init airflow-run airflow-local-test airflow-stop
+
+airflow-docker-test-with-env: develop airflow-clean airflow-init airflow-run airflow-docker-test airflow-stop
+
+engine-slow-test:
+	pytest -m "(fast or slow) and engine"
+
+engine-docker-test:
+	pytest -m "docker and engine"
+
+engine-remote-test:
+	pytest -m "remote and engine"
+
+engine-test:
+	pytest -m "engine"
+
+dbt-test:
+	pytest -m "dbt"
+
+github-test:
+	pytest -m "github"
+
+jupyter-test:
+	pytest -m "jupyter"
+
+web-test:
+	pytest -m "web"
+
+bigquery-test:
+	pytest -m "bigquery"
+
+databricks-test:
+	pytest -m "databricks"
+
+duckdb-test:
+	pytest -m "duckdb"
+
+mssql-test:
+	pytest -m "mssql"
+
+mysql-test:
+	pytest -m "mysql"
+
+postgres-test:
+	pytest -m "postgres"
+
+redshift-test:
+	pytest -m "redshift"
+
+snowflake-test:
+	pytest -m "snowflake"
+
+spark-test:
+	pytest -m "spark"
+
+spark-pyspark-test:
+	pytest -m "spark_pyspark"
+
+trino-test:
+	pytest -m "trino"

examples/airflow/Makefile

@@ -52,5 +52,5 @@ psql:
 spark-sql:
 	docker exec -it airflow-airflow-worker-1 spark-sql
 
-it-test-docker: decorate-docker-compose
+docker-test: decorate-docker-compose
 	docker-compose up --exit-code-from sqlmesh-tests sqlmesh-tests

examples/airflow/docker_compose_decorator.py

@@ -73,7 +73,7 @@
     "entrypoint": "/bin/bash",
     "command": [
         "-c",
-        "make install-dev && pytest -m 'airflow_integration'",
+        "make install-dev && pytest -m 'airflow and docker'",
     ],
     "image": "airflow-sqlmesh",
     "user": "airflow",

pytest.ini

@@ -1,12 +1,33 @@
 [pytest]
 markers =
-    integration: mark test as an integration test which can take longer to run and require an external environment
-    core_integration: mark test as an integration test that does not require an external environment
-    airflow_integration: mark test as needing the Airflow cluster to run successfully
-    spark: mark test as requiring the PySpark dependency
-    airflow: mark test as requiring the Airflow dependency
-    engine_integration: Engine adapter tests that require external services
-    engine_integration_local: Engine adapter tests that require local Docker containers
+    # Test Type Markers
+    # Tests are ordered from fastest to slowest
+    fast: fast tests (automatically applied if no type markers)
+    slow: slow tests that typically involve interacting with a local DB (like DuckDB)
+    docker: test that involves interacting with a Docker container
+    remote: test that involves interacting with a remote DB
+
+    # Test Domain Markers
+    # default: core functionality
+    airflow: test for Airflow scheduler
+    cli: test for CLI
+    dbt: test for dbt adapter
+    github: test for Github CI/CD bot
+    jupyter: tests for Jupyter integration
+    web: tests for web UI
+    spark_pyspark: test for Spark with PySpark dependency
+    # Engine Adapters
+    engine: test all engine adapters
+    bigquery: test for BigQuery
+    databricks: test for Databricks
+    duckdb: test for DuckDB
+    mssql: test for MSSQL
+    mysql: test for MySQL
+    postgres: test for Postgres
+    redshift: test for Redshift
+    snowflake: test for Snowflake
+    spark: test for Spark
+    trino: test for Trino
 
 # Set this to True to enable logging during tests
 log_cli = False

setup.py

@@ -101,9 +101,10 @@
             "tenacity==8.1.0",
             "types-croniter",
             "types-dateparser",
-            "typing-extensions",
+            "types-python-dateutil",
             "types-pytz",
             "types-requests==2.28.8",
+            "typing-extensions",
         ],
         "dbt": [
             "dbt-core<1.5.0",

tests/conftest.py

@@ -133,6 +133,17 @@ def validate(
             raise NotImplementedError(f"Unknown model_name: {model_name}")
 
 
+def pytest_collection_modifyitems(items, *args, **kwargs):
+    test_type_markers = {"fast", "slow", "docker", "remote"}
+    for item in items:
+        for marker in item.iter_markers():
+            if marker.name in test_type_markers:
+                break
+        else:
+            # if no test type marker is found, assume fast test
+            item.add_marker("fast")
+
+
 # Ignore all local config files
 @pytest.fixture(scope="session", autouse=True)
 def ignore_local_config_files():

tests/core/engine_adapter/test_base.py

@@ -18,6 +18,8 @@
 from sqlmesh.utils.errors import UnsupportedCatalogOperationError
 from tests.core.engine_adapter import to_sql_calls
 
+pytestmark = pytest.mark.engine
+
 
 def test_create_view(make_mocked_engine_adapter: t.Callable):
     adapter = make_mocked_engine_adapter(EngineAdapter)

tests/core/engine_adapter/test_base_postgres.py

@@ -2,10 +2,13 @@
 import typing as t
 from unittest.mock import call
 
+import pytest
 from sqlglot import exp, parse_one
 
 from sqlmesh.core.engine_adapter.base_postgres import BasePostgresEngineAdapter
 
+pytestmark = [pytest.mark.engine, pytest.mark.postgres, pytest.mark.redshift]
+
 
 def test_columns(make_mocked_engine_adapter: t.Callable):
     adapter = make_mocked_engine_adapter(BasePostgresEngineAdapter)