From 6380e05cba7807e9b0ca77ab24e2f592e70470bc Mon Sep 17 00:00:00 2001 From: z3z1ma Date: Thu, 31 Jul 2025 12:13:22 -0700 Subject: [PATCH] feat: handle additional pseudocolumn types in bigquery when resolving external table schemas --- sqlmesh/core/engine_adapter/bigquery.py | 28 ++++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/sqlmesh/core/engine_adapter/bigquery.py b/sqlmesh/core/engine_adapter/bigquery.py index adb4aa0d19..e953f4d1d0 100644 --- a/sqlmesh/core/engine_adapter/bigquery.py +++ b/sqlmesh/core/engine_adapter/bigquery.py @@ -324,14 +324,26 @@ def create_mapping_schema( bq_table = self._get_table(table) columns = create_mapping_schema(bq_table.schema) - if ( - include_pseudo_columns - and bq_table.time_partitioning - and not bq_table.time_partitioning.field - ): - columns["_PARTITIONTIME"] = exp.DataType.build("TIMESTAMP", dialect="bigquery") - if bq_table.time_partitioning.type_ == "DAY": - columns["_PARTITIONDATE"] = exp.DataType.build("DATE") + if include_pseudo_columns: + if bq_table.time_partitioning and not bq_table.time_partitioning.field: + columns["_PARTITIONTIME"] = exp.DataType.build("TIMESTAMP", dialect="bigquery") + if bq_table.time_partitioning.type_ == "DAY": + columns["_PARTITIONDATE"] = exp.DataType.build("DATE") + if bq_table.table_id.endswith("*"): + columns["_TABLE_SUFFIX"] = exp.DataType.build("STRING", dialect="bigquery") + if ( + bq_table.external_data_configuration is not None + and bq_table.external_data_configuration.source_format + in ( + "CSV", + "NEWLINE_DELIMITED_JSON", + "AVRO", + "PARQUET", + "ORC", + "DATASTORE_BACKUP", + ) + ): + columns["_FILE_NAME"] = exp.DataType.build("STRING", dialect="bigquery") return columns