Feat!: Create physical tables as part of evaluation (#5189)

Author: izeigerman

docs/cloud/tcloud_getting_started.md

@@ -268,9 +268,6 @@ Models needing backfill (missing dates):
 ├── sqlmesh_example.incremental_model: 2020-01-01 - 2024-11-24
 └── sqlmesh_example.seed_model: 2024-11-24 - 2024-11-24
 Apply - Backfill Tables [y/n]: y
-Creating physical tables ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 3/3 • 0:00:00
-
-All model versions have been created successfully
 
 [1/1] sqlmesh_example.seed_model evaluated in 0.00s
 [1/1] sqlmesh_example.incremental_model evaluated in 0.01s

docs/concepts/macros/macro_variables.md

@@ -130,8 +130,8 @@ SQLMesh provides additional predefined variables used to modify model behavior b
 
 * @runtime_stage - A string value denoting the current stage of the SQLMesh runtime. Typically used in models to conditionally execute pre/post-statements (learn more [here](../models/sql_models.md#optional-prepost-statements)). It returns one of these values:
     * 'loading' - The project is being loaded into SQLMesh's runtime context.
-    * 'creating' - The model tables are being created.
-    * 'evaluating' - The model query logic is being evaluated.
+    * 'creating' - The model tables are being created for the first time. The data may be inserted during table creation.
+    * 'evaluating' - The model query logic is evaluated, and the data is inserted into the existing model table.
     * 'promoting' - The model is being promoted in the target environment (view created during virtual layer update).
     * 'demoting' - The model is being demoted in the target environment (view dropped during virtual layer update).
     * 'auditing' - The audit is being run.

docs/examples/incremental_time_full_walkthrough.md

@@ -304,10 +304,6 @@ Models needing backfill (missing dates):
 Enter the backfill start date (eg. '1 year', '2020-01-01') or blank to backfill from the beginning of history:
 Enter the backfill end date (eg. '1 month ago', '2020-01-01') or blank to backfill up until now:
 Apply - Backfill Tables [y/n]: y
-Creating physical table ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 1/1 • 0:00:07
-
-All model versions have been created successfully
-
 [1/1] demo__dev.incrementals_demo evaluated in 6.97s
 Evaluating models ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 1/1 • 0:00:06
 
@@ -640,9 +636,10 @@ Models:
     ├── tcloud_raw_data.product_usage
     └── tcloud_raw_data.sales
 Apply - Virtual Update [y/n]: y
-Creating physical tables ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 15/15 • 0:00:03
 
-All model versions have been created successfully
+SKIP: No physical layer updates to perform
+
+SKIP: No model batches to execute
 
 Virtually Updating 'prod' ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 0:00:02
 
@@ -732,10 +729,6 @@ Models needing backfill (missing dates):
 Enter the preview start date (eg. '1 year', '2020-01-01') or blank to backfill to preview starting from yesterday: 2024-10-27
 Enter the preview end date (eg. '1 month ago', '2020-01-01') or blank to preview up until '2024-11-08 00:00:00':
 Apply - Preview Tables [y/n]: y
-Creating physical table ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 1/1 • 0:00:16
-
-All model versions have been created successfully
-
 [1/1] demo__dev.incrementals_demo evaluated in 6.18s
 Evaluating models ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 1/1 • 0:00:06
 
@@ -1249,9 +1242,10 @@ Models:
      THEN 'Regular User'
 Directly Modified: demo.incrementals_demo (Forward-only)
 Apply - Virtual Update [y/n]: y
-Creating physical tables ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 15/15 • 0:00:02
 
-All model versions have been created successfully
+SKIP: No physical layer updates to perform
+
+SKIP: No model batches to execute
 
 Virtually Updating 'prod' ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 0:00:02
 

docs/integrations/dlt.md

@@ -102,10 +102,6 @@ Models needing backfill (missing dates):
 ├── sushi_dataset_sqlmesh.incremental_sushi_types: 2024-10-03 - 2024-10-03
 └── sushi_dataset_sqlmesh.incremental_waiters: 2024-10-03 - 2024-10-03
 Apply - Backfill Tables [y/n]: y
-Creating physical table ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% • 3/3 • 0:00:00
-
-All model versions have been created successfully
-
 [1/1] sushi_dataset_sqlmesh.incremental__dlt_loads evaluated in 0.01s
 [1/1] sushi_dataset_sqlmesh.incremental_sushi_types evaluated in 0.00s
 [1/1] sushi_dataset_sqlmesh.incremental_waiters evaluated in 0.01s

examples/multi/repo_1/models/a.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name bronze.a
+  name bronze.a,
+  kind FULL
 );
 
 SELECT

examples/multi/repo_1/models/b.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name bronze.b
+  name bronze.b,
+  kind FULL
 );
 
 SELECT

examples/multi/repo_2/models/c.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name silver.c
+  name silver.c,
+  kind FULL
 );
 
 SELECT DISTINCT col_a

examples/multi/repo_2/models/d.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name silver.d
+  name silver.d,
+  kind FULL
 );
 
 SELECT

examples/multi/repo_2/models/e.sql

@@ -1,5 +1,6 @@
 MODEL (
-  name silver.e
+  name silver.e,
+  kind FULL
 );
 
 SELECT

examples/sushi/config.py

@@ -1,6 +1,6 @@
 import os
 
-from sqlmesh.core.config.common import VirtualEnvironmentMode
+from sqlmesh.core.config.common import VirtualEnvironmentMode, TableNamingConvention
 from sqlmesh.core.config import (
     AutoCategorizationMode,
     BigQueryConnectionConfig,
@@ -27,6 +27,11 @@
 defaults = {"dialect": "duckdb"}
 model_defaults = ModelDefaultsConfig(**defaults)
 model_defaults_iceberg = ModelDefaultsConfig(**defaults, storage_format="iceberg")
+before_all = [
+    "CREATE SCHEMA IF NOT EXISTS raw",
+    "DROP VIEW IF EXISTS raw.demographics",
+    "CREATE VIEW raw.demographics AS (SELECT 1 AS customer_id, '00000' AS zip)",
+]
 
 
 # A DuckDB config, in-memory by default.
@@ -52,6 +57,7 @@
             "nomissingexternalmodels",
         ],
     ),
+    before_all=before_all,
 )
 
 bigquery_config = Config(
@@ -63,6 +69,7 @@
     },
     default_gateway="bq",
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 # A configuration used for SQLMesh tests.
@@ -75,6 +82,7 @@
         )
     ),
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 # A configuration used for SQLMesh tests with virtual environment mode set to DEV_ONLY.
@@ -84,14 +92,15 @@
         "plan": PlanConfig(
             auto_categorize_changes=CategorizerConfig.all_full(),
         ),
-    }
+    },
 )
 
 # A DuckDB config with a physical schema map.
 map_config = Config(
     default_connection=DuckDBConnectionConfig(),
     physical_schema_mapping={"^sushi$": "company_internal"},
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 # A config representing isolated systems with a gateway per system
@@ -103,6 +112,7 @@
     },
     default_gateway="dev",
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 required_approvers_config = Config(
@@ -137,19 +147,21 @@
         ),
     ],
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 
 environment_suffix_table_config = Config(
     default_connection=DuckDBConnectionConfig(),
     model_defaults=model_defaults,
     environment_suffix_target=EnvironmentSuffixTarget.TABLE,
+    before_all=before_all,
 )
 
 environment_suffix_catalog_config = environment_suffix_table_config.model_copy(
     update={
         "environment_suffix_target": EnvironmentSuffixTarget.CATALOG,
-    }
+    },
 )
 
 CATALOGS = {
@@ -161,6 +173,7 @@
     default_connection=DuckDBConnectionConfig(catalogs=CATALOGS),
     default_test_connection=DuckDBConnectionConfig(catalogs=CATALOGS),
     model_defaults=model_defaults,
+    before_all=before_all,
 )
 
 environment_catalog_mapping_config = Config(
@@ -177,4 +190,13 @@
         "^prod$": "prod_catalog",
         ".*": "dev_catalog",
     },
+    before_all=before_all,
+)
+
+hash_md5_naming_config = config.copy(
+    update={"physical_table_naming_convention": TableNamingConvention.HASH_MD5}
+)
+
+table_only_naming_config = config.copy(
+    update={"physical_table_naming_convention": TableNamingConvention.TABLE_ONLY}
 )