Merge remote-tracking branch 'upstream/main' into feat/add-fabric-engine

Author: mattiasthalen

docs/examples/incremental_time_full_walkthrough.md

@@ -172,7 +172,7 @@ We have data like the below that gets ingested into our data warehouse on a dail
 
 I can answer some of the questions above by walking through the model's config, coupled with the business logic/code I prepared ahead of time.
 
-You can see this code in a SQLMesh project context [here](https://github.com/sungchun12/sqlmesh-demos/blob/incremental-demo/models/examples/incrementals_demo.sql).
+You can see this code in a SQLMesh project context [here](https://github.com/sungchun12/sqlmesh-demos/blob/incremental-demo/models/examples/incremental_model.sql).
 
 ```sql
 MODEL (

docs/guides/linter.md

@@ -221,7 +221,7 @@ This example specifies that the model `docs_example.full_model` should not run t
 ```sql linenums="1"
 MODEL(
   name docs_example.full_model,
-  ignored_rules: ["invalidselectstarexpansion"] # or "ALL" to turn off linting completely
+  ignored_rules ["invalidselectstarexpansion"] # or "ALL" to turn off linting completely
 );
 ```
 

docs/guides/ui.md

@@ -1,5 +1,10 @@
 # Browser UI guide
 
+!!! warning
+
+    Browser UI is deprecated. Please use the [VSCode extension](vscode.md) instead.
+
+
 SQLMesh's free, open-source browser user interface (UI) makes it easy to understand, explore, and modify your SQLMesh project.
 
 This page describes the UI's components and how they work.

docs/quickstart/ui.md

@@ -1,5 +1,9 @@
 # Browser UI
 
+!!! warning
+
+    Browser UI is deprecated. Please use the [VSCode extension](../guides/vscode.md) instead.
+
 In this quickstart, you'll use the SQLMesh browser user interface to get up and running with SQLMesh's scaffold generator. This example project will run locally on your computer using [DuckDB](https://duckdb.org/) as an embedded SQL engine.
 
 ??? info "Learn more about the quickstart project structure"

pnpm-lock.yaml

@@ -11,7 +11,7 @@ dependencies = [
     "click",
     "croniter",
     "duckdb>=0.10.0,!=0.10.3",
-    "dateparser",
+    "dateparser<=1.2.1",
     "hyperscript>=0.1.0",
     "importlib-metadata; python_version<'3.12'",
     "ipywidgets",

pyproject.toml

@@ -107,6 +107,14 @@ def cli(
     if "--help" in sys.argv:
         return
 
+    configure_logging(
+        debug,
+        log_to_stdout,
+        log_file_dir=log_file_dir,
+        ignore_warnings=ignore_warnings,
+    )
+    configure_console(ignore_warnings=ignore_warnings)
+
     load = True
 
     if len(paths) == 1:
@@ -117,14 +125,6 @@ def cli(
         if ctx.invoked_subcommand in SKIP_LOAD_COMMANDS:
             load = False
 
-    configure_logging(
-        debug,
-        log_to_stdout,
-        log_file_dir=log_file_dir,
-        ignore_warnings=ignore_warnings,
-    )
-    configure_console(ignore_warnings=ignore_warnings)
-
     configs = load_configs(config, Context.CONFIG_TYPE, paths, dotenv_path=dotenv)
     log_limit = list(configs.values())[0].log_limit
 
@@ -884,6 +884,13 @@ def info(obj: Context, skip_connection: bool, verbose: int) -> None:
 @cli_analytics
 def ui(ctx: click.Context, host: str, port: int, mode: str) -> None:
     """Start a browser-based SQLMesh UI."""
+    from sqlmesh.core.console import get_console
+
+    get_console().log_warning(
+        "The UI is deprecated and will be removed in a future version. Please use the SQLMesh VSCode extension instead. "
+        "Learn more at https://sqlmesh.readthedocs.io/en/stable/guides/vscode/"
+    )
+
     try:
         import uvicorn
     except ModuleNotFoundError as e:

sqlmesh/cli/main.py

@@ -142,3 +142,56 @@ def _normalize_decimal_value(self, col: exp.Expression, precision: int) -> exp.E
             exp.cast(col, "DOUBLE"),
             f"DECIMAL(38, {precision})",
         )
+
+    def _create_table(
+        self,
+        table_name_or_schema: t.Union[exp.Schema, TableName],
+        expression: t.Optional[exp.Expression],
+        exists: bool = True,
+        replace: bool = False,
+        columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
+        table_description: t.Optional[str] = None,
+        column_descriptions: t.Optional[t.Dict[str, str]] = None,
+        table_kind: t.Optional[str] = None,
+        **kwargs: t.Any,
+    ) -> None:
+        catalog = self.get_current_catalog()
+        catalog_type_tuple = self.fetchone(
+            exp.select("type")
+            .from_("duckdb_databases()")
+            .where(exp.column("database_name").eq(catalog))
+        )
+        catalog_type = catalog_type_tuple[0] if catalog_type_tuple else None
+
+        partitioned_by_exps = None
+        if catalog_type == "ducklake":
+            partitioned_by_exps = kwargs.pop("partitioned_by", None)
+
+        super()._create_table(
+            table_name_or_schema,
+            expression,
+            exists,
+            replace,
+            columns_to_types,
+            table_description,
+            column_descriptions,
+            table_kind,
+            **kwargs,
+        )
+
+        if partitioned_by_exps:
+            # Schema object contains column definitions, so we extract Table
+            table_name = (
+                table_name_or_schema.this
+                if isinstance(table_name_or_schema, exp.Schema)
+                else table_name_or_schema
+            )
+            table_name_str = (
+                table_name.sql(dialect=self.dialect)
+                if isinstance(table_name, exp.Table)
+                else table_name
+            )
+            partitioned_by_str = ", ".join(
+                expr.sql(dialect=self.dialect) for expr in partitioned_by_exps
+            )
+            self.execute(f"ALTER TABLE {table_name_str} SET PARTITIONED BY ({partitioned_by_str});")

sqlmesh/core/engine_adapter/duckdb.py

@@ -84,19 +84,8 @@ def to_range(self, read_file: t.Optional[t.List[str]]) -> Range:
         )
 
 
-def read_range_from_file(file: Path, text_range: Range) -> str:
-    """
-    Read the file and return the content within the specified range.
-
-    Args:
-        file: Path to the file to read
-        text_range: The range of text to extract
-
-    Returns:
-        The content within the specified range
-    """
-    with file.open("r", encoding="utf-8") as f:
-        lines = f.readlines()
+def read_range_from_string(content: str, text_range: Range) -> str:
+    lines = content.splitlines(keepends=False)
 
     # Ensure the range is within bounds
     start_line = max(0, text_range.start.line)
@@ -116,6 +105,23 @@ def read_range_from_file(file: Path, text_range: Range) -> str:
     return "".join(result)
 
 
+def read_range_from_file(file: Path, text_range: Range) -> str:
+    """
+    Read the file and return the content within the specified range.
+
+    Args:
+        file: Path to the file to read
+        text_range: The range of text to extract
+
+    Returns:
+        The content within the specified range
+    """
+    with file.open("r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+    return read_range_from_string("".join(lines), text_range)
+
+
 def get_range_of_model_block(
     sql: str,
     dialect: str,

sqlmesh/core/linter/helpers.py

@@ -2,6 +2,7 @@
 
 import abc
 from dataclasses import dataclass
+from pathlib import Path
 
 from sqlmesh.core.model import Model
 
@@ -43,6 +44,7 @@ class Range:
 class TextEdit:
     """A text edit to apply to a file."""
 
+    path: Path
     range: Range
     new_text: str