Merge branch 'main' into feat/add-fabric-engine

Author: mattiasthalen

docs/concepts/macros/sqlmesh_macros.md

@@ -216,18 +216,35 @@ MODEL (
   name @customer.some_table,
   kind FULL,
   blueprints (
-    (customer := customer1, field_a := x, field_b := y),
-    (customer := customer2, field_a := z, field_b := w)
+    (customer := customer1, field_a := x, field_b := y, field_c := 'foo'),
+    (customer := customer2, field_a := z, field_b := w, field_c := 'bar')
   )
 );
 
 SELECT
   @field_a,
-  @{field_b} AS field_b
+  @{field_b} AS field_b,
+  @field_c AS @{field_c}
 FROM @customer.some_source
+
+/*
+When rendered for customer1.some_table:
+SELECT
+  x,
+  y AS field_b,
+  'foo' AS foo
+FROM customer1.some_source
+
+When rendered for customer2.some_table:
+SELECT
+  z,
+  w AS field_b,
+  'bar' AS bar
+FROM customer2.some_source
+*/
 ```
 
-Note the use of both regular `@field_a` and curly brace syntax `@{field_b}` macro variable references in the model query. Learn more [above](#embedding-variables-in-strings)
+Note the use of both regular `@field_a` and curly brace syntax `@{field_b}` macro variable references in the model query. Both of these will be rendered as identifiers. In the case of `field_c`, which in the blueprints is a string, it would be rendered as a string literal when used with the regular macro syntax `@field_c` and if we want to use the string as an identifier then we use the curly braces `@{field_c}`. Learn more [above](#embedding-variables-in-strings)
 
 Blueprint variables can be accessed using the syntax shown above, or through the `@BLUEPRINT_VAR()` macro function, which also supports specifying default values in case the variable is undefined (similar to `@VAR()`).
 

docs/integrations/github.md

@@ -293,13 +293,13 @@ Below is an example of how to define the default config for the bot in either YA
 | `enable_deploy_command`               | Indicates if the `/deploy` command should be enabled in order to allowed synchronized deploys to production. Default: `False`                                                                                                                                                                                                                                                                            |  bool  |    N     |
 | `command_namespace`                   | The namespace to use for SQLMesh commands. For example if you provide `#SQLMesh` as a value then commands will be expected in the format of `#SQLMesh/<command>`. Default: `None` meaning no namespace is used.                                                                                                                                                                                          | string |    N     |
 | `auto_categorize_changes`             | Auto categorization behavior to use for the bot. If not provided then the project-wide categorization behavior is used. See [Auto-categorize model changes](https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#auto-categorize-model-changes) for details.                                                                                                                                   |  dict  |    N     |
-| `default_pr_start`                    | Default start when creating PR environment plans. If running in a mode where the bot automatically backfills models (based on `auto_categorize_changes` behavior) then this can be used to limit the amount of data backfilled. Defaults to `None` meaning the start date is set to the earliest model's start or to 1 day ago if [data previews](../concepts/plans.md#data-preview) need to be computed. |  str   |    N     |
+| `default_pr_start`                    | Default start when creating PR environment plans. If running in a mode where the bot automatically backfills models (based on `auto_categorize_changes` behavior) then this can be used to limit the amount of data backfilled. Defaults to `None` meaning the start date is set to the earliest model's start or to 1 day ago if [data previews](../concepts/plans.md#data-preview) need to be computed.|  str   |    N     |
 | `pr_min_intervals`                    | Intended for use when `default_pr_start` is set to a relative time, eg `1 week ago`. This ensures that at least this many intervals across every model are included for backfill in the PR environment. Without this, models with an interval unit wider than `default_pr_start` (such as `@monthly` models if `default_pr_start` was set to `1 week ago`) will be excluded from backfill entirely.      |  int   |    N     |
 | `skip_pr_backfill`                    | Indicates if the bot should skip backfilling models in the PR environment. Default: `True`                                                                                                                                                                                                                                                                                                               |  bool  |    N     |
 | `pr_include_unmodified`               | Indicates whether to include unmodified models in the PR environment. Default to the project's config value (which defaults to `False`)                                                                                                                                                                                                                                                                  |  bool  |    N     |
 | `run_on_deploy_to_prod`               | Indicates whether to run latest intervals when deploying to prod. If set to false, the deployment will backfill only the changed models up to the existing latest interval in production, ignoring any missing intervals beyond this point. Default: `False`                                                                                                                                             |  bool  |    N     |
 | `pr_environment_name`                 | The name of the PR environment to create for which a PR number will be appended to. Defaults to the repo name if not provided. Note: The name will be normalized to alphanumeric + underscore and lowercase.                                                                                                                                                                                             |  str   |    N     |
-| `prod_branch_name`                    | The name of the git branch associated with production. Ex: `prod`. Default: `main` or `master` is considered prod                                                                                                                                                                                                                                                                                         |  str   |    N     |
+| `prod_branch_name`                    | The name of the git branch associated with production. Ex: `prod`. Default: `main` or `master` is considered prod                                                                                                                                                                                                                                                                                        |  str   |    N     |
 
 Example with all properties defined:
 

sqlmesh/core/config/root.py

@@ -260,6 +260,18 @@ def _normalize_identifiers(key: str) -> None:
 
         return self
 
+    @model_validator(mode="after")
+    def _inherit_project_config_in_cicd_bot(self) -> Self:
+        if self.cicd_bot:
+            # inherit the project-level settings into the CICD bot if they have not been explicitly overridden
+            if self.cicd_bot.auto_categorize_changes_ is None:
+                self.cicd_bot.auto_categorize_changes_ = self.plan.auto_categorize_changes
+
+            if self.cicd_bot.pr_include_unmodified_ is None:
+                self.cicd_bot.pr_include_unmodified_ = self.plan.include_unmodified
+
+        return self
+
     def get_default_test_connection(
         self,
         default_catalog: t.Optional[str] = None,

sqlmesh/core/console.py

@@ -3049,34 +3049,47 @@ class CaptureTerminalConsole(TerminalConsole):
     def __init__(self, console: t.Optional[RichConsole] = None, **kwargs: t.Any) -> None:
         super().__init__(console=console, **kwargs)
         self._captured_outputs: t.List[str] = []
+        self._warnings: t.List[str] = []
         self._errors: t.List[str] = []
 
     @property
     def captured_output(self) -> str:
         return "".join(self._captured_outputs)
 
+    @property
+    def captured_warnings(self) -> str:
+        return "".join(self._warnings)
+
     @property
     def captured_errors(self) -> str:
         return "".join(self._errors)
 
     def consume_captured_output(self) -> str:
-        output = self.captured_output
-        self.clear_captured_outputs()
-        return output
+        try:
+            return self.captured_output
+        finally:
+            self._captured_outputs = []
 
-    def consume_captured_errors(self) -> str:
-        errors = self.captured_errors
-        self.clear_captured_errors()
-        return errors
+    def consume_captured_warnings(self) -> str:
+        try:
+            return self.captured_warnings
+        finally:
+            self._warnings = []
 
-    def clear_captured_outputs(self) -> None:
-        self._captured_outputs = []
+    def consume_captured_errors(self) -> str:
+        try:
+            return self.captured_errors
+        finally:
+            self._errors = []
 
-    def clear_captured_errors(self) -> None:
-        self._errors = []
+    def log_warning(self, short_message: str, long_message: t.Optional[str] = None) -> None:
+        if short_message not in self._warnings:
+            self._warnings.append(short_message)
+        super().log_warning(short_message, long_message)
 
     def log_error(self, message: str) -> None:
-        self._errors.append(message)
+        if message not in self._errors:
+            self._errors.append(message)
         super().log_error(message)
 
     def log_skipped_models(self, snapshot_names: t.Set[str]) -> None:
@@ -3087,9 +3100,8 @@ def log_skipped_models(self, snapshot_names: t.Set[str]) -> None:
             super().log_skipped_models(snapshot_names)
 
     def log_failed_models(self, errors: t.List[NodeExecutionFailedError]) -> None:
-        if errors:
-            self._errors.append("\n".join(str(ex) for ex in errors))
-            super().log_failed_models(errors)
+        self._errors.extend([str(ex) for ex in errors if str(ex) not in self._errors])
+        super().log_failed_models(errors)
 
     def _print(self, value: t.Any, **kwargs: t.Any) -> None:
         with self.console.capture() as capture:
@@ -3110,6 +3122,11 @@ class MarkdownConsole(CaptureTerminalConsole):
     AUDIT_PADDING = 7
 
     def __init__(self, **kwargs: t.Any) -> None:
+        self.alert_block_max_content_length = int(kwargs.pop("alert_block_max_content_length", 500))
+        self.alert_block_collapsible_threshold = int(
+            kwargs.pop("alert_block_collapsible_threshold", 200)
+        )
+
         super().__init__(
             **{**kwargs, "console": RichConsole(no_color=True, width=kwargs.pop("width", None))}
         )
@@ -3434,18 +3451,40 @@ def show_linter_violations(
         self._print(msg)
         self._errors.append(msg)
 
-    def log_error(self, message: str) -> None:
-        super().log_error(f"```\n\\[ERROR] {message}```\n\n")
+    @property
+    def captured_warnings(self) -> str:
+        return self._render_alert_block("WARNING", self._warnings)
 
-    def log_warning(self, short_message: str, long_message: t.Optional[str] = None) -> None:
-        logger.warning(long_message or short_message)
+    @property
+    def captured_errors(self) -> str:
+        return self._render_alert_block("CAUTION", self._errors)
 
-        if not short_message.endswith("\n"):
-            short_message += (
-                "\n"  # so that the closing ``` ends up on a newline which is important for GitHub
-            )
+    def _render_alert_block(self, block_type: str, items: t.List[str]) -> str:
+        # GitHub Markdown alert syntax, https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts
+        if items:
+            item_contents = ""
+            list_indicator = "- " if len(items) > 1 else ""
+
+            for item in items:
+                item = item.replace("\n", "\n> ")
+                item_contents += f">\n> {list_indicator}{item}\n"
+
+                if len(item_contents) > self.alert_block_max_content_length:
+                    truncation_msg = (
+                        "...\n>\n> Truncated. Please check the console for full information.\n"
+                    )
+                    item_contents = item_contents[
+                        0 : self.alert_block_max_content_length - len(truncation_msg)
+                    ]
+                    item_contents += truncation_msg
+                    break
+
+            if len(item_contents) > self.alert_block_collapsible_threshold:
+                item_contents = f"> <details>\n{item_contents}> </details>"
+
+            return f"> [!{block_type}]\n{item_contents}\n"
 
-        self._print(f"```\n\\[WARNING] {short_message}```\n\n")
+        return ""
 
     def _print(self, value: t.Any, **kwargs: t.Any) -> None:
         self.console.print(value, **kwargs)
@@ -3472,7 +3511,7 @@ def _print(self, value: t.Any, **kwargs: t.Any) -> None:
         super()._print(value, **kwargs)
         for captured_output in self._captured_outputs:
             print(captured_output)
-        self.clear_captured_outputs()
+        self.consume_captured_output()
 
     def _prompt(self, message: str, **kwargs: t.Any) -> t.Any:
         self._print(message)

sqlmesh/integrations/github/cicd/command.py

@@ -119,10 +119,7 @@ def _update_pr_environment(controller: GithubController) -> bool:
     except Exception as e:
         logger.exception("Error occurred when updating PR environment")
         conclusion = controller.update_pr_environment_check(
-            status=GithubCheckStatus.COMPLETED,
-            exception=e,
-            plan=controller.pr_plan_or_none,
-            plan_flags=controller.pr_plan_flags,
+            status=GithubCheckStatus.COMPLETED, exception=e
         )
         return (
             conclusion is not None

sqlmesh/integrations/github/cicd/config.py

@@ -7,6 +7,7 @@
 from sqlmesh.core.config.base import BaseConfig
 from sqlmesh.utils.date import TimeLike
 from sqlmesh.utils.pydantic import model_validator
+from sqlmesh.core.console import get_console
 
 
 class MergeMethod(str, Enum):
@@ -22,10 +23,12 @@ class GithubCICDBotConfig(BaseConfig):
     enable_deploy_command: bool = False
     merge_method: t.Optional[MergeMethod] = None
     command_namespace: t.Optional[str] = None
-    auto_categorize_changes: CategorizerConfig = CategorizerConfig.all_off()
+    auto_categorize_changes_: t.Optional[CategorizerConfig] = Field(
+        default=None, alias="auto_categorize_changes"
+    )
     default_pr_start: t.Optional[TimeLike] = None
-    skip_pr_backfill: bool = True
-    pr_include_unmodified: t.Optional[bool] = None
+    skip_pr_backfill_: t.Optional[bool] = Field(default=None, alias="skip_pr_backfill")
+    pr_include_unmodified_: t.Optional[bool] = Field(default=None, alias="pr_include_unmodified")
     run_on_deploy_to_prod: bool = False
     pr_environment_name: t.Optional[str] = None
     pr_min_intervals: t.Optional[int] = None
@@ -50,6 +53,26 @@ def prod_branch_names(self) -> t.List[str]:
             return [self.prod_branch_names_]
         return ["main", "master"]
 
+    @property
+    def auto_categorize_changes(self) -> CategorizerConfig:
+        return self.auto_categorize_changes_ or CategorizerConfig.all_off()
+
+    @property
+    def pr_include_unmodified(self) -> bool:
+        return self.pr_include_unmodified_ or False
+
+    @property
+    def skip_pr_backfill(self) -> bool:
+        if self.skip_pr_backfill_ is None:
+            get_console().log_warning(
+                "`skip_pr_backfill` is unset, defaulting it to `true` (no data will be backfilled).\n"
+                "Future versions of SQLMesh will default to `skip_pr_backfill: false` to align with the CLI default behaviour.\n"
+                "If you would like to preserve the current behaviour and remove this warning, please explicitly set `skip_pr_backfill: true` in the bot config.\n\n"
+                "For more information on configuring the bot, see: https://sqlmesh.readthedocs.io/en/stable/integrations/github/"
+            )
+            return True
+        return self.skip_pr_backfill_
+
     FIELDS_FOR_ANALYTICS: t.ClassVar[t.Set[str]] = {
         "invalidate_environment_after_deploy",
         "enable_deploy_command",