Actually run spark tests

Author: erindru

.circleci/install-prerequisites.sh

@@ -34,4 +34,9 @@ echo "Installing OS-level dependencies: $ALL_DEPENDENCIES"
 
 sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES
 
+if [ "$ENGINE" == "spark" ]; then
+    echo "Using Java version for spark:"
+    java -version
+fi
+
 echo "All done"

Makefile

@@ -188,7 +188,7 @@ postgres-test: engine-postgres-up
 	pytest -n auto -m "postgres" --retries 3 --junitxml=test-results/junit-postgres.xml
 
 spark-test: engine-spark-up
-	pytest -n auto -m "spark" --retries 3 --junitxml=test-results/junit-spark.xml
+	pytest -n auto -m "spark" --retries 3 --junitxml=test-results/junit-spark.xml && pytest -n auto -m "pyspark" --retries 3 --junitxml=test-results/junit-pyspark.xml
 
 trino-test: engine-trino-up
 	pytest -n auto -m "trino" --retries 3 --junitxml=test-results/junit-trino.xml

pyproject.toml

@@ -264,6 +264,7 @@ markers = [
     "redshift: test for Redshift",
     "snowflake: test for Snowflake",
     "spark: test for Spark",
+    "pyspark: test for PySpark that need to run separately from the other spark tests",
     "trino: test for Trino (all connectors)",
     "risingwave: test for Risingwave",
 

tests/engines/spark/test_db_api.py

@@ -4,7 +4,10 @@
 from sqlmesh.engines.spark.db_api import errors
 from sqlmesh.engines.spark.db_api import spark_session as spark_session_db
 
-pytestmark = [pytest.mark.slow, pytest.mark.spark]
+# note: this is deliberately not marked with 'spark' so that it
+# can run separately from the spark integration tests.
+# running them at the same time mutates some global state in the SparkSession which breaks these tests
+pytestmark = [pytest.mark.slow, pytest.mark.pyspark]
 
 
 def test_spark_session_cursor(spark_session: SparkSession):