Skip to content

fix behaviour of VACUUM for vector indices to make rowid consistent between shadow tables and base table #239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
coco-speed wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix behaviour of VACUUM for vector indices to make rowid consistent between shadow tables and base table #239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
76558e8
fix behaviour of VACUUM for vector indices to make rowid consistent
sivukhin Aug 6, 2024
853143d
build bundles
sivukhin Aug 6, 2024
2115277
fix bug
sivukhin Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 28 additions & 27 deletions libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
** src/test2.c
** src/test3.c
** src/test8.c
** src/vacuum.c
** src/vdbe.c
** src/vdbeInt.h
** src/vdbeapi.c
Expand Down Expand Up @@ -155950,6 +155951,10 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate(
/* #include "sqliteInt.h" */
/* #include "vdbeInt.h" */

#ifndef SQLITE_OMIT_VECTOR
/* #include "vectorIndexInt.h" */
#endif

#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)

/*
Expand Down Expand Up @@ -156227,6 +156232,27 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
if( rc!=SQLITE_OK ) goto end_of_vacuum;
db->init.iDb = 0;

#ifndef SQLITE_OMIT_VECTOR
// shadow tables for vector index will be populated automatically during CREATE INDEX command
// so we must skip them at this step
if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
rc = execSqlF(db, pzErrMsg,
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
"||' SELECT*FROM\"%w\".'||quote(name)"
"FROM vacuum_db.sqlite_schema "
"WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
zDbMain
);
}else{
rc = execSqlF(db, pzErrMsg,
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
"||' SELECT*FROM\"%w\".'||quote(name)"
"FROM vacuum_db.sqlite_schema "
"WHERE type='table'AND coalesce(rootpage,1)>0",
zDbMain
);
}
#else
/* Loop through the tables in the main database. For each, do
** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
** the contents to the temporary database.
Expand All @@ -156238,6 +156264,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
"WHERE type='table'AND coalesce(rootpage,1)>0",
zDbMain
);
#endif
assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
db->mDbFlags &= ~DBFLAG_Vacuum;
if( rc!=SQLITE_OK ) goto end_of_vacuum;
Expand Down Expand Up @@ -213656,11 +213683,6 @@ int vectorF64ParseSqliteBlob(
** VectorIdxParams utilities
****************************************************************************/

// VACUUM creates tables and indices first and only then populate data
// we need to ignore inserts from 'INSERT INTO vacuum.t SELECT * FROM t' statements because
// all shadow tables will be populated by VACUUM process during regular process of table copy
#define IsVacuum(db) ((db->mDbFlags&DBFLAG_Vacuum)!=0)

void vectorIdxParamsInit(VectorIdxParams *pParams, u8 *pBinBuf, int nBinSize) {
assert( nBinSize <= VECTOR_INDEX_PARAMS_BUF_SIZE );

Expand Down Expand Up @@ -214379,10 +214401,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
// this is done to prevent unrecoverable situations where index were dropped but index parameters deletion failed and second attempt will fail on first step
int rcIdx, rcParams;

if( IsVacuum(db) ){
return SQLITE_OK;
}

assert( zDbSName != NULL );

rcIdx = diskAnnDropIndex(db, zDbSName, zIdxName);
Expand All @@ -214393,10 +214411,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
assert( zDbSName != NULL );

if( IsVacuum(db) ){
return SQLITE_OK;
}

return diskAnnClearIndex(db, zDbSName, zIdxName);
}

Expand All @@ -214406,7 +214420,7 @@ int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
* this made intentionally in order to natively support upload of SQLite dumps
*
* dump populates tables first and create indices after
* so we must omit them because shadow tables already filled
* so we must omit index refill setp because shadow tables already filled
*
* 1. in case of any error :-1 returned (and pParse errMsg is populated with some error message)
* 2. if vector index must not be created : 0 returned
Expand All @@ -214424,10 +214438,6 @@ int vectorIndexCreate(Parse *pParse, const Index *pIdx, const char *zDbSName, co
int hasLibsqlVectorIdxFn = 0, hasCollation = 0;
const char *pzErrMsg;

if( IsVacuum(pParse->db) ){
return CREATE_IGNORE;
}

assert( zDbSName != NULL );

sqlite3 *db = pParse->db;
Expand Down Expand Up @@ -214577,7 +214587,6 @@ int vectorIndexSearch(
VectorIdxParams idxParams;
vectorIdxParamsInit(&idxParams, NULL, 0);

assert( !IsVacuum(db) );
assert( zDbSName != NULL );

if( argc != 3 ){
Expand Down Expand Up @@ -214662,10 +214671,6 @@ int vectorIndexInsert(
int rc;
VectorInRow vectorInRow;

if( IsVacuum(pCur->db) ){
return SQLITE_OK;
}

rc = vectorInRowAlloc(pCur->db, pRecord, &vectorInRow, pzErrMsg);
if( rc != SQLITE_OK ){
return rc;
Expand All @@ -214685,10 +214690,6 @@ int vectorIndexDelete(
){
VectorInRow payload;

if( IsVacuum(pCur->db) ){
return SQLITE_OK;
}

payload.pVector = NULL;
payload.nKeys = r->nField - 1;
payload.pKeyValues = r->aMem + 1;
Expand Down
26 changes: 23 additions & 3 deletions libsql-ffi/bundled/bindings/bindgen.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ extern "C" {
}

pub const __GNUC_VA_LIST: i32 = 1;
pub const SQLITE_VERSION: &[u8; 7] = b"3.44.0\0";
pub const SQLITE_VERSION_NUMBER: i32 = 3044000;
pub const SQLITE_VERSION: &[u8; 7] = b"3.45.1\0";
pub const SQLITE_VERSION_NUMBER: i32 = 3045001;
pub const SQLITE_SOURCE_ID: &[u8; 85] =
b"2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad8alt1\0";
b"2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257ccalt1\0";
pub const LIBSQL_VERSION: &[u8; 6] = b"0.2.3\0";
pub const SQLITE_OK: i32 = 0;
pub const SQLITE_ERROR: i32 = 1;
Expand Down Expand Up @@ -356,6 +356,7 @@ pub const SQLITE_DETERMINISTIC: i32 = 2048;
pub const SQLITE_DIRECTONLY: i32 = 524288;
pub const SQLITE_SUBTYPE: i32 = 1048576;
pub const SQLITE_INNOCUOUS: i32 = 2097152;
pub const SQLITE_RESULT_SUBTYPE: i32 = 16777216;
pub const SQLITE_WIN32_DATA_DIRECTORY_TYPE: i32 = 1;
pub const SQLITE_WIN32_TEMP_DIRECTORY_TYPE: i32 = 2;
pub const SQLITE_TXN_NONE: i32 = 0;
Expand Down Expand Up @@ -408,6 +409,7 @@ pub const SQLITE_TESTCTRL_PENDING_BYTE: i32 = 11;
pub const SQLITE_TESTCTRL_ASSERT: i32 = 12;
pub const SQLITE_TESTCTRL_ALWAYS: i32 = 13;
pub const SQLITE_TESTCTRL_RESERVE: i32 = 14;
pub const SQLITE_TESTCTRL_JSON_SELFCHECK: i32 = 14;
pub const SQLITE_TESTCTRL_OPTIMIZATIONS: i32 = 15;
pub const SQLITE_TESTCTRL_ISKEYWORD: i32 = 16;
pub const SQLITE_TESTCTRL_SCRATCHMALLOC: i32 = 17;
Expand Down Expand Up @@ -3133,6 +3135,24 @@ pub struct Fts5ExtensionApi {
piCol: *mut ::std::os::raw::c_int,
),
>,
pub xQueryToken: ::std::option::Option<
unsafe extern "C" fn(
arg1: *mut Fts5Context,
iPhrase: ::std::os::raw::c_int,
iToken: ::std::os::raw::c_int,
ppToken: *mut *const ::std::os::raw::c_char,
pnToken: *mut ::std::os::raw::c_int,
) -> ::std::os::raw::c_int,
>,
pub xInstToken: ::std::option::Option<
unsafe extern "C" fn(
arg1: *mut Fts5Context,
iIdx: ::std::os::raw::c_int,
iToken: ::std::os::raw::c_int,
arg2: *mut *const ::std::os::raw::c_char,
arg3: *mut ::std::os::raw::c_int,
) -> ::std::os::raw::c_int,
>,
}
#[repr(C)]
#[derive(Debug, Copy, Clone)]
Expand Down
55 changes: 28 additions & 27 deletions libsql-ffi/bundled/src/sqlite3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
** src/test2.c
** src/test3.c
** src/test8.c
** src/vacuum.c
** src/vdbe.c
** src/vdbeInt.h
** src/vdbeapi.c
Expand Down Expand Up @@ -155950,6 +155951,10 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate(
/* #include "sqliteInt.h" */
/* #include "vdbeInt.h" */

#ifndef SQLITE_OMIT_VECTOR
/* #include "vectorIndexInt.h" */
#endif

#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)

/*
Expand Down Expand Up @@ -156227,6 +156232,27 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
if( rc!=SQLITE_OK ) goto end_of_vacuum;
db->init.iDb = 0;

#ifndef SQLITE_OMIT_VECTOR
// shadow tables for vector index will be populated automatically during CREATE INDEX command
// so we must skip them at this step
if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
rc = execSqlF(db, pzErrMsg,
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
"||' SELECT*FROM\"%w\".'||quote(name)"
"FROM vacuum_db.sqlite_schema "
"WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
zDbMain
);
}else{
rc = execSqlF(db, pzErrMsg,
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
"||' SELECT*FROM\"%w\".'||quote(name)"
"FROM vacuum_db.sqlite_schema "
"WHERE type='table'AND coalesce(rootpage,1)>0",
zDbMain
);
}
#else
/* Loop through the tables in the main database. For each, do
** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
** the contents to the temporary database.
Expand All @@ -156238,6 +156264,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
"WHERE type='table'AND coalesce(rootpage,1)>0",
zDbMain
);
#endif
assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
db->mDbFlags &= ~DBFLAG_Vacuum;
if( rc!=SQLITE_OK ) goto end_of_vacuum;
Expand Down Expand Up @@ -213656,11 +213683,6 @@ int vectorF64ParseSqliteBlob(
** VectorIdxParams utilities
****************************************************************************/

// VACUUM creates tables and indices first and only then populate data
// we need to ignore inserts from 'INSERT INTO vacuum.t SELECT * FROM t' statements because
// all shadow tables will be populated by VACUUM process during regular process of table copy
#define IsVacuum(db) ((db->mDbFlags&DBFLAG_Vacuum)!=0)

void vectorIdxParamsInit(VectorIdxParams *pParams, u8 *pBinBuf, int nBinSize) {
assert( nBinSize <= VECTOR_INDEX_PARAMS_BUF_SIZE );

Expand Down Expand Up @@ -214379,10 +214401,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
// this is done to prevent unrecoverable situations where index were dropped but index parameters deletion failed and second attempt will fail on first step
int rcIdx, rcParams;

if( IsVacuum(db) ){
return SQLITE_OK;
}

assert( zDbSName != NULL );

rcIdx = diskAnnDropIndex(db, zDbSName, zIdxName);
Expand All @@ -214393,10 +214411,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
assert( zDbSName != NULL );

if( IsVacuum(db) ){
return SQLITE_OK;
}

return diskAnnClearIndex(db, zDbSName, zIdxName);
}

Expand All @@ -214406,7 +214420,7 @@ int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
* this made intentionally in order to natively support upload of SQLite dumps
*
* dump populates tables first and create indices after
* so we must omit them because shadow tables already filled
* so we must omit index refill setp because shadow tables already filled
*
* 1. in case of any error :-1 returned (and pParse errMsg is populated with some error message)
* 2. if vector index must not be created : 0 returned
Expand All @@ -214424,10 +214438,6 @@ int vectorIndexCreate(Parse *pParse, const Index *pIdx, const char *zDbSName, co
int hasLibsqlVectorIdxFn = 0, hasCollation = 0;
const char *pzErrMsg;

if( IsVacuum(pParse->db) ){
return CREATE_IGNORE;
}

assert( zDbSName != NULL );

sqlite3 *db = pParse->db;
Expand Down Expand Up @@ -214577,7 +214587,6 @@ int vectorIndexSearch(
VectorIdxParams idxParams;
vectorIdxParamsInit(&idxParams, NULL, 0);

assert( !IsVacuum(db) );
assert( zDbSName != NULL );

if( argc != 3 ){
Expand Down Expand Up @@ -214662,10 +214671,6 @@ int vectorIndexInsert(
int rc;
VectorInRow vectorInRow;

if( IsVacuum(pCur->db) ){
return SQLITE_OK;
}

rc = vectorInRowAlloc(pCur->db, pRecord, &vectorInRow, pzErrMsg);
if( rc != SQLITE_OK ){
return rc;
Expand All @@ -214685,10 +214690,6 @@ int vectorIndexDelete(
){
VectorInRow payload;

if( IsVacuum(pCur->db) ){
return SQLITE_OK;
}

payload.pVector = NULL;
payload.nKeys = r->nField - 1;
payload.pKeyValues = r->aMem + 1;
Expand Down
26 changes: 26 additions & 0 deletions libsql-sqlite3/src/vacuum.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
#include "sqliteInt.h"
#include "vdbeInt.h"

#ifndef SQLITE_OMIT_VECTOR
#include "vectorIndexInt.h"
#endif

#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)

/*
Expand Down Expand Up @@ -294,6 +298,27 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
if( rc!=SQLITE_OK ) goto end_of_vacuum;
db->init.iDb = 0;

#ifndef SQLITE_OMIT_VECTOR
// shadow tables for vector index will be populated automatically during CREATE INDEX command
// so we must skip them at this step
if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
rc = execSqlF(db, pzErrMsg,
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
"||' SELECT*FROM\"%w\".'||quote(name)"
"FROM vacuum_db.sqlite_schema "
"WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
zDbMain
);
}else{
rc = execSqlF(db, pzErrMsg,
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
"||' SELECT*FROM\"%w\".'||quote(name)"
"FROM vacuum_db.sqlite_schema "
"WHERE type='table'AND coalesce(rootpage,1)>0",
zDbMain
);
}
#else
/* Loop through the tables in the main database. For each, do
** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
** the contents to the temporary database.
Expand All @@ -305,6 +330,7 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
"WHERE type='table'AND coalesce(rootpage,1)>0",
zDbMain
);
#endif
assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
db->mDbFlags &= ~DBFLAG_Vacuum;
if( rc!=SQLITE_OK ) goto end_of_vacuum;
Expand Down
Loading