feat: Add support for Microsoft Fabric Warehouse (#4751)

Co-authored-by: Mattias Thalén <bitter-polders0x@icloud.com>
Co-authored-by: Erin Drummond <erin.dru@gmail.com>

Author: 3 people

.circleci/continue_config.yml

@@ -304,6 +304,8 @@ workflows:
                 - bigquery
                 - clickhouse-cloud
                 - athena
+                # todo: enable fabric when cicd catalog create/drop implemented in manage-test-db.sh
+                #- fabric
                 - gcp-postgres
           filters:
             branches:

.circleci/install-prerequisites.sh

@@ -12,7 +12,7 @@ fi
 
 ENGINE="$1"
 
-COMMON_DEPENDENCIES="libpq-dev netcat-traditional"
+COMMON_DEPENDENCIES="libpq-dev netcat-traditional unixodbc-dev"
 ENGINE_DEPENDENCIES=""
 
 if [ "$ENGINE" == "spark" ]; then

Makefile

@@ -173,6 +173,9 @@ clickhouse-cloud-test: guard-CLICKHOUSE_CLOUD_HOST guard-CLICKHOUSE_CLOUD_USERNA
 athena-test: guard-AWS_ACCESS_KEY_ID guard-AWS_SECRET_ACCESS_KEY guard-ATHENA_S3_WAREHOUSE_LOCATION engine-athena-install
 	pytest -n auto -m "athena" --retries 3 --junitxml=test-results/junit-athena.xml
 
+fabric-test: guard-FABRIC_HOST guard-FABRIC_CLIENT_ID guard-FABRIC_CLIENT_SECRET guard-FABRIC_DATABASE engine-fabric-install
+	pytest -n auto -m "fabric" --retries 3 --junitxml=test-results/junit-fabric.xml	
+
 gcp-postgres-test: guard-GCP_POSTGRES_INSTANCE_CONNECTION_STRING guard-GCP_POSTGRES_USER guard-GCP_POSTGRES_PASSWORD guard-GCP_POSTGRES_KEYFILE_JSON engine-gcppostgres-install
 	pytest -n auto -m "gcp_postgres" --retries 3 --junitxml=test-results/junit-gcp-postgres.xml
 

docs/guides/configuration.md

@@ -909,6 +909,7 @@ These pages describe the connection configuration options for each execution eng
 * [BigQuery](../integrations/engines/bigquery.md)
 * [Databricks](../integrations/engines/databricks.md)
 * [DuckDB](../integrations/engines/duckdb.md)
+* [Fabric](../integrations/engines/fabric.md)
 * [MotherDuck](../integrations/engines/motherduck.md)
 * [MySQL](../integrations/engines/mysql.md)
 * [MSSQL](../integrations/engines/mssql.md)

docs/integrations/engines/fabric.md

@@ -0,0 +1,34 @@
+# Fabric
+
+## Local/Built-in Scheduler
+**Engine Adapter Type**: `fabric`
+
+NOTE: Fabric Warehouse is not recommended to be used for the SQLMesh [state connection](../../reference/configuration.md#connections).
+
+### Installation
+#### Microsoft Entra ID / Azure Active Directory Authentication:
+```
+pip install "sqlmesh[fabric]"
+```
+
+### Connection options
+
+| Option            | Description                                                  |     Type     | Required |
+| ----------------- | ------------------------------------------------------------ | :----------: | :------: |
+| `type`            | Engine type name - must be `fabric`                           |    string    |    Y     |
+| `host`            | The hostname of the Fabric Warehouse server                             |    string    |    Y     |
+| `user`            | The client id to use for authentication with the Fabric Warehouse server |    string    |    N     |
+| `password`        | The client secret to use for authentication with the Fabric Warehouse server |    string    |    N     |
+| `port`            | The port number of the Fabric Warehouse server                          |     int      |    N     |
+| `database`        | The target database                                          |    string    |    N     |
+| `charset`         | The character set used for the connection                    |    string    |    N     |
+| `timeout`         | The query timeout in seconds. Default: no timeout            |     int      |    N     |
+| `login_timeout`   | The timeout for connection and login in seconds. Default: 60 |     int      |    N     |
+| `appname`         | The application name to use for the connection               |    string    |    N     |
+| `conn_properties` | The list of connection properties                            | list[string] |    N     |
+| `autocommit`      | Is autocommit mode enabled. Default: false                   |     bool     |    N     |
+| `driver`          | The driver to use for the connection. Default: pyodbc            |    string    |    N     |
+| `driver_name`     | The driver name to use for the connection. E.g., *ODBC Driver 18 for SQL Server* |    string    |    N     |
+| `tenant_id`          | The Azure / Entra tenant UUID                             |    string    |    Y     |
+| `workspace_id`       | The Fabric workspace UUID. The preferred way to retrieve it is by running `notebookutils.runtime.context.get("currentWorkspaceId")` in a python notebook.                             |    string    |    Y     |
+| `odbc_properties` | The dict of ODBC connection properties. E.g., authentication: ActiveDirectoryServicePrincipal. See more [here](https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?view=sql-server-ver16). | dict |    N     |

docs/integrations/overview.md

@@ -17,6 +17,7 @@ SQLMesh supports the following execution engines for running SQLMesh projects (e
 * [ClickHouse](./engines/clickhouse.md) (clickhouse)
 * [Databricks](./engines/databricks.md) (databricks)
 * [DuckDB](./engines/duckdb.md) (duckdb)
+* [Fabric](./engines/fabric.md) (fabric)
 * [MotherDuck](./engines/motherduck.md) (motherduck)
 * [MSSQL](./engines/mssql.md) (mssql)
 * [MySQL](./engines/mysql.md) (mysql)

mkdocs.yml

@@ -83,6 +83,7 @@ nav:
       - integrations/engines/clickhouse.md
       - integrations/engines/databricks.md
       - integrations/engines/duckdb.md
+      - integrations/engines/fabric.md
       - integrations/engines/motherduck.md
       - integrations/engines/mssql.md
       - integrations/engines/mysql.md

pyproject.toml

@@ -103,6 +103,7 @@ dev = [
 dbt = ["dbt-core<2"]
 dlt = ["dlt"]
 duckdb = []
+fabric = ["pyodbc>=5.0.0"]
 gcppostgres = ["cloud-sql-python-connector[pg8000]>=1.8.0"]
 github = ["PyGithub>=2.6.0"]
 llm = ["langchain", "openai"]
@@ -253,6 +254,7 @@ markers = [
     "clickhouse_cloud: test for Clickhouse (cloud mode)",
     "databricks: test for Databricks",
     "duckdb: test for DuckDB",
+    "fabric: test for Fabric",
     "motherduck: test for MotherDuck",
     "mssql: test for MSSQL",
     "mysql: test for MySQL",

sqlmesh/core/config/__init__.py

@@ -13,6 +13,7 @@
     ConnectionConfig as ConnectionConfig,
     DatabricksConnectionConfig as DatabricksConnectionConfig,
     DuckDBConnectionConfig as DuckDBConnectionConfig,
+    FabricConnectionConfig as FabricConnectionConfig,
     GCPPostgresConnectionConfig as GCPPostgresConnectionConfig,
     MotherDuckConnectionConfig as MotherDuckConnectionConfig,
     MSSQLConnectionConfig as MSSQLConnectionConfig,

sqlmesh/core/config/connection.py

@@ -43,7 +43,13 @@
 
 logger = logging.getLogger(__name__)
 
-RECOMMENDED_STATE_SYNC_ENGINES = {"postgres", "gcp_postgres", "mysql", "mssql", "azuresql"}
+RECOMMENDED_STATE_SYNC_ENGINES = {
+    "postgres",
+    "gcp_postgres",
+    "mysql",
+    "mssql",
+    "azuresql",
+}
 FORBIDDEN_STATE_SYNC_ENGINES = {
     # Do not support row-level operations
     "spark",
@@ -1684,6 +1690,55 @@ def _extra_engine_config(self) -> t.Dict[str, t.Any]:
         return {"catalog_support": CatalogSupport.SINGLE_CATALOG_ONLY}
 
 
+class FabricConnectionConfig(MSSQLConnectionConfig):
+    """
+    Fabric Connection Configuration.
+    Inherits most settings from MSSQLConnectionConfig and sets the type to 'fabric'.
+    It is recommended to use the 'pyodbc' driver for Fabric.
+    """
+
+    type_: t.Literal["fabric"] = Field(alias="type", default="fabric")  # type: ignore
+    DIALECT: t.ClassVar[t.Literal["fabric"]] = "fabric"  # type: ignore
+    DISPLAY_NAME: t.ClassVar[t.Literal["Fabric"]] = "Fabric"  # type: ignore
+    DISPLAY_ORDER: t.ClassVar[t.Literal[17]] = 17  # type: ignore
+    driver: t.Literal["pyodbc"] = "pyodbc"
+    workspace_id: str
+    tenant_id: str
+    autocommit: t.Optional[bool] = True
+
+    @property
+    def _engine_adapter(self) -> t.Type[EngineAdapter]:
+        from sqlmesh.core.engine_adapter.fabric import FabricEngineAdapter
+
+        return FabricEngineAdapter
+
+    @property
+    def _connection_factory(self) -> t.Callable:
+        # Override to support catalog switching for Fabric
+        base_factory = super()._connection_factory
+
+        def create_fabric_connection(
+            target_catalog: t.Optional[str] = None, *args: t.Any, **kwargs: t.Any
+        ) -> t.Callable:
+            kwargs["database"] = target_catalog or self.database
+            return base_factory(*args, **kwargs)
+
+        return create_fabric_connection
+
+    @property
+    def _extra_engine_config(self) -> t.Dict[str, t.Any]:
+        return {
+            "database": self.database,
+            # more operations than not require a specific catalog to be already active
+            # in particular, create/drop view, create/drop schema and querying information_schema
+            "catalog_support": CatalogSupport.REQUIRES_SET_CATALOG,
+            "workspace_id": self.workspace_id,
+            "tenant_id": self.tenant_id,
+            "user": self.user,
+            "password": self.password,
+        }
+
+
 class SparkConnectionConfig(ConnectionConfig):
     """
     Vanilla Spark Connection Configuration. Use `DatabricksConnectionConfig` for Databricks.