From 443e8dd86d429c76c88742eeb5bc799cc1e3ed0e Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Sun, 7 Sep 2025 11:26:39 +0100 Subject: [PATCH 1/2] Support parameters in sqlite_query This PR adds new optional `params` argument to `sqlite_query` function that allows to pass query parameters to an underlying SQLite prepared statement. Parameters are specified as a `STRUCT`, that can be created inline using `row()` function: ```sql SELECT * from sqlite_query(sakila, 'SELECT ? || ? a', params=row('foo', 'bar')) ---- foobar ``` When the whole DuckDB SQL statement, that contains `sqlite_query()` function, is used with `PREPARE` + `EXECUTE` (for example, from Python or Java client), then the external client-provided parameters will be forwarded to SQLite: ```sql PREPARE p1 as SELECT * from sqlite_query(sakila, 'SELECT ? || ? a', params=row(?, ?)) EXECUTE p1('foo', 'bar') ---- foobar EXECUTE p1('baz', 'boo') ---- bazboo DEALLOCATE p1 ``` The following parameter types are supported: `BIGINT`, `DOUBLE`, `BLOB`, `VARCHAR`. Only positional parameters are supported - `STRUCT` field names are ignored. Testing: existing `sqlite_query` test is updated with parameters checks. --- src/include/sqlite_scanner.hpp | 1 + src/include/sqlite_stmt.hpp | 3 + src/sqlite_scanner.cpp | 5 ++ src/sqlite_stmt.cpp | 31 +++++++++++ src/storage/sqlite_query.cpp | 29 ++++++++++ test/sql/storage/attach_sqlite_query.test | 68 +++++++++++++++++++++++ 6 files changed, 137 insertions(+) diff --git a/src/include/sqlite_scanner.hpp b/src/include/sqlite_scanner.hpp index 9bfe700..243cdfc 100644 --- a/src/include/sqlite_scanner.hpp +++ b/src/include/sqlite_scanner.hpp @@ -22,6 +22,7 @@ struct SqliteBindData : public TableFunctionData { vector names; vector types; string sql; + vector params; RowIdInfo row_id_info; bool all_varchar = false; diff --git a/src/include/sqlite_stmt.hpp b/src/include/sqlite_stmt.hpp index d9b8674..b737e49 100644 --- a/src/include/sqlite_stmt.hpp +++ b/src/include/sqlite_stmt.hpp @@ -41,8 +41,11 @@ class SQLiteStatement { throw InternalException("Unsupported type for SQLiteStatement::Bind"); } void BindText(idx_t col, const string_t &value); + void BindText(idx_t col, const string &value); void BindBlob(idx_t col, const string_t &value); + void BindBlob(idx_t col, const string &value); void BindValue(Vector &col, idx_t c, idx_t r); + void BindParameter(const Value ¶m, idx_t param_idx); int GetType(idx_t col); string GetName(idx_t col); idx_t GetColumnCount(); diff --git a/src/sqlite_scanner.cpp b/src/sqlite_scanner.cpp index eb8b5ff..568595a 100644 --- a/src/sqlite_scanner.cpp +++ b/src/sqlite_scanner.cpp @@ -123,6 +123,11 @@ static void SqliteInitInternal(ClientContext &context, const SqliteBindData &bin sql = bind_data.sql; } local_state.stmt = local_state.db->Prepare(sql.c_str()); + + for (idx_t i = 0; i < bind_data.params.size(); i++) { + const Value ¶m = bind_data.params[i]; + local_state.stmt.BindParameter(param, i); + } } static unique_ptr SqliteCardinality(ClientContext &context, const FunctionData *bind_data_p) { diff --git a/src/sqlite_stmt.cpp b/src/sqlite_stmt.cpp index ee618b0..8237295 100644 --- a/src/sqlite_stmt.cpp +++ b/src/sqlite_stmt.cpp @@ -148,10 +148,18 @@ void SQLiteStatement::BindBlob(idx_t col, const string_t &value) { SQLiteUtils::Check(sqlite3_bind_blob(stmt, col + 1, value.GetDataUnsafe(), value.GetSize(), nullptr), db); } +void SQLiteStatement::BindBlob(idx_t col, const string &value) { + SQLiteUtils::Check(sqlite3_bind_blob(stmt, col + 1, value.c_str(), value.length(), nullptr), db); +} + void SQLiteStatement::BindText(idx_t col, const string_t &value) { SQLiteUtils::Check(sqlite3_bind_text(stmt, col + 1, value.GetDataUnsafe(), value.GetSize(), nullptr), db); } +void SQLiteStatement::BindText(idx_t col, const string &value) { + SQLiteUtils::Check(sqlite3_bind_text(stmt, col + 1, value.c_str(), value.length(), nullptr), db); +} + template <> void SQLiteStatement::Bind(idx_t col, std::nullptr_t value) { SQLiteUtils::Check(sqlite3_bind_null(stmt, col + 1), db); @@ -181,4 +189,27 @@ void SQLiteStatement::BindValue(Vector &col, idx_t c, idx_t r) { } } +void SQLiteStatement::BindParameter(const Value ¶m, idx_t param_idx) { + if (param.IsNull()) { + Bind(param_idx, nullptr); + } else { + switch (param.type().id()) { + case LogicalTypeId::BIGINT: + Bind(param_idx, BigIntValue::Get(param)); + break; + case LogicalTypeId::DOUBLE: + Bind(param_idx, DoubleValue::Get(param)); + break; + case LogicalTypeId::BLOB: + BindBlob(param_idx, StringValue::Get(param)); + break; + case LogicalTypeId::VARCHAR: + BindText(param_idx, StringValue::Get(param)); + break; + default: + throw InternalException("Unsupported parameter type \"%s\", index: %zu for SQLite::BindValue", param.type().ToString(), param_idx); + } + } +} + } // namespace duckdb diff --git a/src/storage/sqlite_query.cpp b/src/storage/sqlite_query.cpp index 061de36..e776f5c 100644 --- a/src/storage/sqlite_query.cpp +++ b/src/storage/sqlite_query.cpp @@ -42,6 +42,33 @@ static unique_ptr SQLiteQueryBind(ClientContext &context, TableFun StringUtil::RTrim(sql); } + vector params; + auto params_it = input.named_parameters.find("params"); + if (params_it != input.named_parameters.end()) { + Value &struct_val = params_it->second; + if (struct_val.IsNull()) { + throw BinderException("Parameters to sqlite_query cannot be NULL"); + } + if (struct_val.type().id() != LogicalTypeId::STRUCT) { + throw BinderException("Query parameters must be specified in a STRUCT"); + } + params = StructValue::GetChildren(struct_val); + for (idx_t i = 0; i < params.size(); i++) { + const Value ¶m = params[i]; + switch(param.type().id()) { + case LogicalTypeId::BIGINT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::BLOB: + case LogicalTypeId::VARCHAR: + break; + default: + if (!param.IsNull()) { + throw BinderException("Unsupported parameter type \"%s\", index: %zu", param.type().ToString(), i); + } + } + } + } + auto &con = transaction.GetDB(); auto stmt = con.Prepare(sql); if (!stmt.stmt) { @@ -57,6 +84,7 @@ static unique_ptr SQLiteQueryBind(ClientContext &context, TableFun } result->rows_per_group = optional_idx(); result->sql = std::move(sql); + result->params = std::move(params); result->all_varchar = true; result->file_name = sqlite_catalog.GetDBPath(); result->global_db = &con; @@ -70,5 +98,6 @@ SQLiteQueryFunction::SQLiteQueryFunction() init_local = scan_function.init_local; function = scan_function.function; global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE; + named_parameters["params"] = LogicalType::ANY; } } // namespace duckdb diff --git a/test/sql/storage/attach_sqlite_query.test b/test/sql/storage/attach_sqlite_query.test index 01cd6e4..71efc18 100644 --- a/test/sql/storage/attach_sqlite_query.test +++ b/test/sql/storage/attach_sqlite_query.test @@ -17,6 +17,11 @@ SELECT * FROM sqlite_query(s, 'SELECT unixepoch(''1992-01-01'') a') ---- 694224000 +query I +SELECT * FROM sqlite_query(s, 'SELECT unixepoch(?) a', params=row('1992-01-01')) +---- +694224000 + statement ok CREATE OR REPLACE TABLE s.tbl AS SELECT * FROM range(10000) t(r); @@ -40,6 +45,13 @@ GUINESS ED GUINESS PENELOPE GUINESS SEAN +query II +SELECT last_name, first_name FROM sqlite_query(sakila, 'SELECT * FROM actor WHERE last_name=?', params=row('GUINESS')) ORDER BY ALL +---- +GUINESS ED +GUINESS PENELOPE +GUINESS SEAN + statement error SELECT * FROM sqlite_query(s, '') ---- @@ -54,3 +66,59 @@ statement error SELECT * FROM sqlite_query(s, 'CREATE TABLE my_table(a,b,c)') ---- query must return data + +query I +SELECT * from sqlite_query(sakila, 'SELECT ? a', params=row(NULL)) +---- +NULL + +query I +SELECT * from sqlite_query(sakila, 'SELECT ? a', params=row(42::BIGINT)) +---- +42 + +query I +SELECT * from sqlite_query(sakila, 'SELECT ? a', params=row(42.123::DOUBLE)) +---- +42.123 + +query I +SELECT * from sqlite_query(sakila, 'SELECT ? a', params=row('foo'::BLOB)) +---- +foo + +query I +SELECT * from sqlite_query(sakila, 'SELECT ? || ? a', params=row('foo', 'bar')) +---- +foobar + +statement ok +PREPARE p1 as SELECT * from sqlite_query(sakila, 'SELECT ? || ? a', params=row(?, ?)) + +query I +EXECUTE p1('foo', 'bar') +---- +foobar + +query I +EXECUTE p1('baz', 'boo') +---- +bazboo + +statement ok +DEALLOCATE p1 + +statement error +SELECT * from sqlite_query(sakila, 'SELECT ? a', params=row(42::TINYINT)) +---- +Binder Error: Unsupported parameter type "TINYINT", index: 0 + +statement error +SELECT * from sqlite_query(sakila, 'SELECT ? a', params=NULL) +---- +Binder Error: Parameters to sqlite_query cannot be NULL + +statement error +SELECT * from sqlite_query(sakila, 'SELECT ? a', params=42) +---- +Binder Error: Query parameters must be specified in a STRUCT From c4efd66e8243e64edbacade4ad4c97d50b8b24df Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Fri, 26 Sep 2025 13:57:57 +0100 Subject: [PATCH 2/2] Update extension-ci-tools --- .github/workflows/MainDistributionPipeline.yml | 4 ++-- extension-ci-tools | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 299872f..c98f93b 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,7 +14,7 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.0 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: duckdb_version: main extension_name: sqlite_scanner @@ -23,7 +23,7 @@ jobs: duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.4.0 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main secrets: inherit with: duckdb_version: main diff --git a/extension-ci-tools b/extension-ci-tools index ee7f51d..ba18d4f 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit ee7f51d06562bbea87d6f6f921def85557e44d18 +Subproject commit ba18d4f106a6cc1d5597f442bac06a1d7db098ef