From 79c041bf37df9aad9cdd9c01af6518e07994521f Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Tue, 19 Aug 2025 23:26:06 +0000 Subject: [PATCH 1/5] Chore(fabric): Enable integration tests --- .circleci/continue_config.yml | 13 ++++++------- .circleci/manage-test-db.sh | 19 +++++++++++++++++++ docs/integrations/engines/fabric.md | 3 +++ .../engine_adapter/integration/config.yaml | 1 + 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 5fca61d7ed..6c7bcdea44 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -303,14 +303,13 @@ workflows: - redshift - bigquery - clickhouse-cloud - - athena - # todo: enable fabric when cicd catalog create/drop implemented in manage-test-db.sh - #- fabric + - athena + - fabric - gcp-postgres - filters: - branches: - only: - - main + #filters: + # branches: + # only: + # - main - ui_style - ui_test - vscode_test diff --git a/.circleci/manage-test-db.sh b/.circleci/manage-test-db.sh index 80ca075912..f44bd54845 100755 --- a/.circleci/manage-test-db.sh +++ b/.circleci/manage-test-db.sh @@ -133,7 +133,26 @@ gcp-postgres_down() { gcp-postgres_exec "drop database $1" } +# Fabric +fabric_init() { + python --version #note: as at 2025-08-20, ms-fabric-cli is pinned to Python >= 3.10, <3.13 + pip install ms-fabric-cli + + # to prevent the '[EncryptionFailed] An error occurred with the encrypted cache.' error + # ref: https://microsoft.github.io/fabric-cli/#switch-to-interactive-mode-optional + fab config set encryption_fallback_enabled true + echo "Logging in to Fabric" + fab auth login -u $FABRIC_CLIENT_ID -p $FABRIC_CLIENT_SECRET --tenant $FABRIC_TENANT_ID +} + +fabric_up() { + fab create "SQLMesh CircleCI.Workspace/$1.Warehouse" +} + +fabric_down() { + fab rm -f "SQLMesh CircleCI.Workspace/$1.Warehouse" || true +} INIT_FUNC="${ENGINE}_init" UP_FUNC="${ENGINE}_up" diff --git a/docs/integrations/engines/fabric.md b/docs/integrations/engines/fabric.md index eb00b5ac1d..90ac3234fc 100644 --- a/docs/integrations/engines/fabric.md +++ b/docs/integrations/engines/fabric.md @@ -1,5 +1,8 @@ # Fabric +!!! info + The Fabric engine adapter is a community contribution. Due to this, only limited community support is available. + ## Local/Built-in Scheduler **Engine Adapter Type**: `fabric` diff --git a/tests/core/engine_adapter/integration/config.yaml b/tests/core/engine_adapter/integration/config.yaml index 51241889de..b75efc762b 100644 --- a/tests/core/engine_adapter/integration/config.yaml +++ b/tests/core/engine_adapter/integration/config.yaml @@ -198,6 +198,7 @@ gateways: workspace_id: {{ env_var("FABRIC_WORKSPACE_ID") }} odbc_properties: Authentication: ActiveDirectoryServicePrincipal + check_import: false state_connection: type: duckdb From c9de5daec04e702cfab32cd4855835f323dd1d1c Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Tue, 19 Aug 2025 23:50:16 +0000 Subject: [PATCH 2/5] Install ms odbc driver in ci --- .circleci/continue_config.yml | 32 +++++++++++++++++------------- .circleci/install-prerequisites.sh | 11 +++++++++- packages-microsoft-prod.deb | 1 + 3 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 packages-microsoft-prod.deb diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 6c7bcdea44..638dc74bce 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -237,6 +237,9 @@ jobs: steps: - halt_unless_core - checkout + - run: + name: Install OS-level dependencies + command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>" - run: name: Generate database name command: | @@ -247,6 +250,7 @@ jobs: echo "export DATABRICKS_CATALOG='$TEST_DB_NAME'" >> "$BASH_ENV" echo "export REDSHIFT_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV" echo "export GCP_POSTGRES_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV" + echo "export FABRIC_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV" - run: name: Create test database command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" up @@ -282,13 +286,13 @@ workflows: parameters: engine: - duckdb - - postgres - - mysql - - mssql - - trino - - spark - - clickhouse - - risingwave + #- postgres + #- mysql + #- mssql + #- trino + #- spark + #- clickhouse + #- risingwave - engine_tests_cloud: name: cloud_engine_<< matrix.engine >> context: @@ -298,14 +302,14 @@ workflows: matrix: parameters: engine: - - snowflake - - databricks - - redshift - - bigquery - - clickhouse-cloud - - athena + #- snowflake + #- databricks + #- redshift + #- bigquery + #- clickhouse-cloud + #- athena - fabric - - gcp-postgres + #- gcp-postgres #filters: # branches: # only: diff --git a/.circleci/install-prerequisites.sh b/.circleci/install-prerequisites.sh index acd25ae02c..cbd8491535 100755 --- a/.circleci/install-prerequisites.sh +++ b/.circleci/install-prerequisites.sh @@ -17,12 +17,21 @@ ENGINE_DEPENDENCIES="" if [ "$ENGINE" == "spark" ]; then ENGINE_DEPENDENCIES="default-jdk" +elif [ "$ENGINE" == "fabric" ]; then + echo "Installing Microsoft package repository" + + # ref: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server + curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + rm packages-microsoft-prod.deb + + ENGINE_DEPENDENCIES="msodbcsql18" fi ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES" echo "Installing OS-level dependencies: $ALL_DEPENDENCIES" -sudo apt-get clean && sudo apt-get -y update && sudo apt-get -y install $ALL_DEPENDENCIES +sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES echo "All done" \ No newline at end of file diff --git a/packages-microsoft-prod.deb b/packages-microsoft-prod.deb new file mode 100644 index 0000000000..1becba2bb0 --- /dev/null +++ b/packages-microsoft-prod.deb @@ -0,0 +1 @@ +404: Not Found \ No newline at end of file From 370e263587b5fa437f133e3ccfa88ea77dbd5c59 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Wed, 20 Aug 2025 00:30:49 +0000 Subject: [PATCH 3/5] Re-enable full test suite --- .circleci/continue_config.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 638dc74bce..155d095844 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -286,13 +286,13 @@ workflows: parameters: engine: - duckdb - #- postgres - #- mysql - #- mssql - #- trino - #- spark - #- clickhouse - #- risingwave + - postgres + - mysql + - mssql + - trino + - spark + - clickhouse + - risingwave - engine_tests_cloud: name: cloud_engine_<< matrix.engine >> context: @@ -302,14 +302,14 @@ workflows: matrix: parameters: engine: - #- snowflake - #- databricks - #- redshift - #- bigquery - #- clickhouse-cloud - #- athena + - snowflake + - databricks + - redshift + - bigquery + - clickhouse-cloud + - athena - fabric - #- gcp-postgres + - gcp-postgres #filters: # branches: # only: From 3ee1c6f9089cdfc7b52bd4fb36696456274216f7 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Wed, 20 Aug 2025 00:34:16 +0000 Subject: [PATCH 4/5] remove accidental package commit --- packages-microsoft-prod.deb | 1 - 1 file changed, 1 deletion(-) delete mode 100644 packages-microsoft-prod.deb diff --git a/packages-microsoft-prod.deb b/packages-microsoft-prod.deb deleted file mode 100644 index 1becba2bb0..0000000000 --- a/packages-microsoft-prod.deb +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file From 9cb105dae5db68236375d092967446c561255d56 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Wed, 20 Aug 2025 00:55:55 +0000 Subject: [PATCH 5/5] Reinstate branch filter --- .circleci/continue_config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 155d095844..8f8324a2a0 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -310,10 +310,10 @@ workflows: - athena - fabric - gcp-postgres - #filters: - # branches: - # only: - # - main + filters: + branches: + only: + - main - ui_style - ui_test - vscode_test