Merge branch 'main' of https://github.com/TobikoData/sqlmesh

Author: sungchun12

.circleci/continue_config.yml

@@ -201,39 +201,6 @@ jobs:
           name: Run tests
           command: npm --prefix web/client run test
 
-  trigger_private_tests:
-    docker:
-      - image: cimg/python:3.12.0
-    resource_class: small
-    steps:
-      - checkout
-      - run:
-          name: Install setuptools scm
-          command: pip install setuptools_scm
-      - run:
-          name: Trigger private tests
-          command: |
-            export COMMIT_MESSAGE="$(git log --format=%s -n 1 $CIRCLE_SHA1)"
-            export FORMATTED_COMMIT_MESSAGE="${COMMIT_MESSAGE//\"/\\\"}"
-            # returns a version string like 0.1.0.dev11
-            export PACKAGE_VERSION="$(python ./.circleci/get_scm_version.py)"
-            curl --request POST \
-              --url $TOBIKO_PRIVATE_CIRCLECI_URL \
-              --header "Circle-Token: $TOBIKO_PRIVATE_CIRCLECI_KEY" \
-              --header "content-type: application/json" \
-              --data '{
-                "branch":"main",
-                "parameters":{
-                  "run_main_pr":false,
-                  "run_sqlmesh_commit":true,
-                  "sqlmesh_branch":"'$CIRCLE_BRANCH'",
-                  "sqlmesh_commit_author":"'$CIRCLE_USERNAME'",
-                  "sqlmesh_commit_hash":"'$CIRCLE_SHA1'",
-                  "sqlmesh_commit_message":"'"$FORMATTED_COMMIT_MESSAGE"'",
-                  "sqlmesh_package_version":"'$PACKAGE_VERSION'"
-                  }
-              }'
-
   engine_tests_docker:
     parameters:
       engine:
@@ -340,13 +307,6 @@ workflows:
             branches:
               only:
                 - main
-      - trigger_private_tests:
-          requires:
-            - style_and_cicd_tests
-          filters:
-            branches:
-              only:
-                - main
       - ui_style
       - ui_test
       - vscode_test

.github/workflows/pr.yaml

@@ -43,7 +43,7 @@ jobs:
         with:
           python-version: '3.12'
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@v6
       - name: Install python dependencies
         run: |
           python -m venv .venv

docs/concepts/macros/macro_variables.md

@@ -132,7 +132,8 @@ SQLMesh provides additional predefined variables used to modify model behavior b
     * 'loading' - The project is being loaded into SQLMesh's runtime context.
     * 'creating' - The model tables are being created.
     * 'evaluating' - The model query logic is being evaluated.
-    * 'promoting' - The model is being promoted in the target environment (virtual layer update).
+    * 'promoting' - The model is being promoted in the target environment (view created during virtual layer update).
+    * 'demoting' - The model is being demoted in the target environment (view dropped during virtual layer update).
     * 'auditing' - The audit is being run.
     * 'testing' - The model query logic is being evaluated in the context of a unit test.
 * @gateway - A string value containing the name of the current [gateway](../../guides/connections.md).

docs/concepts/macros/sqlmesh_macros.md

@@ -216,18 +216,35 @@ MODEL (
   name @customer.some_table,
   kind FULL,
   blueprints (
-    (customer := customer1, field_a := x, field_b := y),
-    (customer := customer2, field_a := z, field_b := w)
+    (customer := customer1, field_a := x, field_b := y, field_c := 'foo'),
+    (customer := customer2, field_a := z, field_b := w, field_c := 'bar')
   )
 );
 
 SELECT
   @field_a,
-  @{field_b} AS field_b
+  @{field_b} AS field_b,
+  @field_c AS @{field_c}
 FROM @customer.some_source
+
+/*
+When rendered for customer1.some_table:
+SELECT
+  x,
+  y AS field_b,
+  'foo' AS foo
+FROM customer1.some_source
+
+When rendered for customer2.some_table:
+SELECT
+  z,
+  w AS field_b,
+  'bar' AS bar
+FROM customer2.some_source
+*/
 ```
 
-Note the use of both regular `@field_a` and curly brace syntax `@{field_b}` macro variable references in the model query. Learn more [above](#embedding-variables-in-strings)
+Note the use of both regular `@field_a` and curly brace syntax `@{field_b}` macro variable references in the model query. Both of these will be rendered as identifiers. In the case of `field_c`, which in the blueprints is a string, it would be rendered as a string literal when used with the regular macro syntax `@field_c` and if we want to use the string as an identifier then we use the curly braces `@{field_c}`. Learn more [above](#embedding-variables-in-strings)
 
 Blueprint variables can be accessed using the syntax shown above, or through the `@BLUEPRINT_VAR()` macro function, which also supports specifying default values in case the variable is undefined (similar to `@VAR()`).
 

docs/concepts/plans.md

@@ -246,6 +246,63 @@ Models needing backfill (missing dates):
 Enter the backfill end date (eg. '1 month ago', '2020-01-01') or blank to backfill up until '2024-09-27 00:00:00':
 ```
 
+#### Minimum intervals
+
+When you run a plan with a fixed `--start` or `--end` date, you create a virtual data environment with a limited subset of data. However, if the time range specified is less than the size of an interval on one of your models, that model will be skipped by default.
+
+For example, if you have a model like so:
+
+```sql
+MODEL(
+    name sqlmesh_example.monthly_model,
+    kind INCREMENTAL_BY_TIME_RANGE (
+        time_column month
+    ),
+    cron '@monthly'
+);
+
+SELECT SUM(a) AS sum_a, MONTH(day) AS month
+FROM sqlmesh_example.upstream_model
+WHERE day BETWEEN @start_ds AND @end_ds
+```
+
+make a change to it and run the following:
+
+```bash linenums="1" hl_lines="8"
+$ sqlmesh plan dev --start '1 day ago' 
+
+Models:
+└── Added:
+    └── sqlmesh_example__dev.monthly_model
+Apply - Virtual Update [y/n]: y
+
+SKIP: No model batches to execute
+```
+
+No data will be backfilled because `1 day ago` does not contain a complete month. However, you can use the `--min-intervals` option to override this behaviour like so:
+
+```bash linenums="1" hl_lines="11"
+$ sqlmesh plan dev --start '1 day ago' --min-intervals 1
+
+Models:
+└── Added:
+    └── sqlmesh_example__dev.monthly_model
+Apply - Virtual Update [y/n]: y
+
+[1/1] sqlmesh_example__dev.monthly_model   [insert 2025-06-01 - 2025-06-30]   0.08s   
+Executing model batches ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 1/1 • 0:00:00                                                             
+                                                                                                                                                    
+✔ Model batches executed
+```
+
+This will ensure that regardless of the plan `--start` date, all added or modified models will have at least `--min-intervals` intervals considered for backfill.
+
+!!! info
+
+    If you are running plans manually you can just adjust the `--start` date to be wide enough to cover the models in question.
+
+    The `--min-intervals` option is primarily intended for [automation scenarios](../integrations/github.md) where the plan is always run with a default relative start date and you always want (for example) "2 weeks worth of data" in the target environment.
+
 ### Data preview for forward-only changes
 As mentioned earlier, the data output produced by [forward-only changes](#forward-only-change) in a development environment can only be used for preview and will not be reused in production.
 

docs/guides/signals.md

@@ -63,9 +63,9 @@ This example specifies that the `random_signal()` should evaluate once with a th
 MODEL (
   name example.signal_model,
   kind FULL,
-  signals [
+  signals (
     random_signal(threshold := 0.5), # specify threshold value
-  ]
+  )
 );
 
 SELECT 1
@@ -108,9 +108,9 @@ MODEL (
     time_column ds,
   ),
   start '2 week ago',
-  signals [
+  signals (
     one_week_ago(),
-  ]
+  )
 );
 
 

docs/guides/vscode.md

@@ -1,5 +1,7 @@
 # Visual Studio Code Extension
 
+<div style="position: relative; padding-bottom: 56.25%; height: 0;"><iframe src="https://www.loom.com/embed/4a2e974ec8294716a4b1dbb0146add82?sid=b6c8def6-b7e0-4bfc-af6c-e37d5d83b0b1" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen style="position: absolute; top: 0; left: 0; width: 100%; height: 100%;"></iframe></div>
+
 !!! danger "Preview"
 
     The SQLMesh Visual Studio Code extension is in preview and undergoing active development. You may encounter bugs or API incompatibilities with the SQLMesh version you are running.

docs/integrations/github.md

@@ -293,12 +293,14 @@ Below is an example of how to define the default config for the bot in either YA
 | `enable_deploy_command`               | Indicates if the `/deploy` command should be enabled in order to allowed synchronized deploys to production. Default: `False`                                                                                                                                                                                                                                                                            |  bool  |    N     |
 | `command_namespace`                   | The namespace to use for SQLMesh commands. For example if you provide `#SQLMesh` as a value then commands will be expected in the format of `#SQLMesh/<command>`. Default: `None` meaning no namespace is used.                                                                                                                                                                                          | string |    N     |
 | `auto_categorize_changes`             | Auto categorization behavior to use for the bot. If not provided then the project-wide categorization behavior is used. See [Auto-categorize model changes](https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#auto-categorize-model-changes) for details.                                                                                                                                   |  dict  |    N     |
-| `default_pr_start`                    | Default start when creating PR environment plans. If running in a mode where the bot automatically backfills models (based on `auto_categorize_changes` behavior) then this can be used to limit the amount of data backfilled. Defaults to `None` meaning the start date is set to the earliest model's start or to 1 day ago if [data previews](../concepts/plans.md#data-preview) need to be computed. |  str   |    N     |
+| `default_pr_start`                    | Default start when creating PR environment plans. If running in a mode where the bot automatically backfills models (based on `auto_categorize_changes` behavior) then this can be used to limit the amount of data backfilled. Defaults to `None` meaning the start date is set to the earliest model's start or to 1 day ago if [data previews](../concepts/plans.md#data-preview) need to be computed.|  str   |    N     |
+| `pr_min_intervals`                    | Intended for use when `default_pr_start` is set to a relative time, eg `1 week ago`. This ensures that at least this many intervals across every model are included for backfill in the PR environment. Without this, models with an interval unit wider than `default_pr_start` (such as `@monthly` models if `default_pr_start` was set to `1 week ago`) will be excluded from backfill entirely.      |  int   |    N     |
 | `skip_pr_backfill`                    | Indicates if the bot should skip backfilling models in the PR environment. Default: `True`                                                                                                                                                                                                                                                                                                               |  bool  |    N     |
 | `pr_include_unmodified`               | Indicates whether to include unmodified models in the PR environment. Default to the project's config value (which defaults to `False`)                                                                                                                                                                                                                                                                  |  bool  |    N     |
 | `run_on_deploy_to_prod`               | Indicates whether to run latest intervals when deploying to prod. If set to false, the deployment will backfill only the changed models up to the existing latest interval in production, ignoring any missing intervals beyond this point. Default: `False`                                                                                                                                             |  bool  |    N     |
 | `pr_environment_name`                 | The name of the PR environment to create for which a PR number will be appended to. Defaults to the repo name if not provided. Note: The name will be normalized to alphanumeric + underscore and lowercase.                                                                                                                                                                                             |  str   |    N     |
-| `prod_branch_name`                    | The name of the git branch associated with production. Ex: `prod`. Default: `main` or `master` is considered prod                                                                                                                                                                                                                                                                                         |  str   |    N     |
+| `prod_branch_name`                    | The name of the git branch associated with production. Ex: `prod`. Default: `main` or `master` is considered prod                                                                                                                                                                                                                                                                                        |  str   |    N     |
+| `forward_only_branch_suffix`          | If the git branch has this suffix, trigger a [forward-only](../concepts/plans.md#forward-only-plans) plan instead of a normal plan. Default: `-forward-only`                                                                                                                                                                                                                                             |  str   |    N     |
 
 Example with all properties defined:
 

docs/reference/cli.md

@@ -149,7 +149,7 @@ Options:
 ```
 Usage: sqlmesh destroy
 
-  Removes all project resources, including warehouse objects, state tables, the SQLMesh cache and any build artifacts.
+  Removes all state tables, the SQLMesh cache and all project resources, including warehouse objects. This includes all tables, views and schemas managed by SQLMesh, as well as any external resources that may have been created by other tools within those schemas.
 
 Options:
   --help               Show this message and exit.

docs/reference/notebook.md

@@ -250,7 +250,7 @@ options:
 ```
 %destroy
 
-Removes all project resources, including warehouse objects, state tables, the SQLMesh cache and any build artifacts.
+Removes all state tables, the SQLMesh cache, and other project resources, including warehouse objects. This includes all tables, views, and schemas managed by SQLMesh, as well as any external resources that may have been created by other tools within those schemas.
 ```
 
 #### dlt_refresh