Ensure that blobs match the length of arrays they're being deserialized into and otherwise fail.

.github/workflows/publish-server.yml

@@ -118,23 +118,9 @@ jobs:
           context: .
           platforms: ${{ env.platform }}
           labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-debug,push-by-digest=true,name-canonical=true,push=true
           build-args: |
             BUILD_DEBUG=true
-      -
-        name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"          
-      -
-        name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-debug-${{ env.PLATFORM_PAIR }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
 
   build-arm64:
     permissions: write-all

.github/workflows/rust.yml

@@ -159,8 +159,8 @@ jobs:
             target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
-      - name: check libsql remote
-        run: cargo check -p libsql --no-default-features -F remote
+      - name: build libsql all features
+        run: cargo build -p libsql --all-features 
 
   # test-rust-wasm:
   #   runs-on: ubuntu-latest

Cargo.toml

@@ -34,7 +34,7 @@ codegen-units = 1
 panic = "unwind"
 
 [workspace.dependencies]
-rusqlite = { package = "libsql-rusqlite", path = "vendored/rusqlite", version = "0.31", default-features = false, features = [
+rusqlite = { package = "libsql-rusqlite", path = "vendored/rusqlite", version = "0.32", default-features = false, features = [
     "libsql-experimental",
     "column_decltype",
     "load_extension",

libsql-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "libsql-ffi"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 build = "build.rs"
 license = "MIT"

libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c

@@ -28,6 +28,7 @@
 **    README.md
 **    configure
 **    configure.ac
+**    ext/fts5/fts5_tokenize.c
 **    ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java
 **    ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java
 **    ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java
@@ -69,6 +70,7 @@
 **    src/test2.c
 **    src/test3.c
 **    src/test8.c
+**    src/vacuum.c
 **    src/vdbe.c
 **    src/vdbeInt.h
 **    src/vdbeapi.c
@@ -155952,6 +155954,10 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate(
 /* #include "sqliteInt.h" */
 /* #include "vdbeInt.h" */
 
+#ifndef SQLITE_OMIT_VECTOR
+/* #include "vectorIndexInt.h" */
+#endif
+
 #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
 
 /*
@@ -156229,6 +156235,27 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
   db->init.iDb = 0;
 
+#ifndef SQLITE_OMIT_VECTOR
+  // shadow tables for vector index will be populated automatically during CREATE INDEX command
+  // so we must skip them at this step
+  if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
+    rc = execSqlF(db, pzErrMsg,
+        "SELECT'INSERT INTO vacuum_db.'||quote(name)"
+        "||' SELECT*FROM\"%w\".'||quote(name)"
+        "FROM vacuum_db.sqlite_schema "
+        "WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
+        zDbMain
+    );
+  }else{
+    rc = execSqlF(db, pzErrMsg,
+        "SELECT'INSERT INTO vacuum_db.'||quote(name)"
+        "||' SELECT*FROM\"%w\".'||quote(name)"
+        "FROM vacuum_db.sqlite_schema "
+        "WHERE type='table'AND coalesce(rootpage,1)>0",
+        zDbMain
+    );
+  }
+#else
   /* Loop through the tables in the main database. For each, do
   ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
   ** the contents to the temporary database.
@@ -156240,6 +156267,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
       "WHERE type='table'AND coalesce(rootpage,1)>0",
       zDbMain
   );
+#endif
   assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
   db->mDbFlags &= ~DBFLAG_Vacuum;
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
@@ -211974,6 +212002,7 @@ int diskAnnCreateIndex(
   int type, dims;
   u64 maxNeighborsParam, blockSizeBytes;
   char *zSql;
+  const char *zRowidColumnName;
   char columnSqlDefs[VECTOR_INDEX_SQL_RENDER_LIMIT]; // definition of columns (e.g. index_key INTEGER BINARY, index_key1 TEXT, ...)
   char columnSqlNames[VECTOR_INDEX_SQL_RENDER_LIMIT]; // just column names (e.g. index_key, index_key1, index_key2, ...)
   if( vectorIdxKeyDefsRender(pKey, "index_key", columnSqlDefs, sizeof(columnSqlDefs)) != 0 ){
@@ -212041,6 +212070,7 @@ int diskAnnCreateIndex(
         columnSqlDefs,
         columnSqlNames
         );
+    zRowidColumnName = "index_key";
   }else{
     zSql = sqlite3MPrintf(
         db,
@@ -212050,9 +212080,31 @@ int diskAnnCreateIndex(
         columnSqlDefs,
         columnSqlNames
         );
+    zRowidColumnName = "rowid";
   }
   rc = sqlite3_exec(db, zSql, 0, 0, 0);
   sqlite3DbFree(db, zSql);
+  if( rc != SQLITE_OK ){
+    return rc;
+  }
+  /*
+   * vector blobs are usually pretty huge (more than a page size, for example, node block for 1024d f32 embeddings with 1bit compression will occupy ~20KB)
+   * in this case, main table B-Tree takes on redundant shape where all leaf nodes has only 1 cell
+   *
+   * as we have a query which selects random row using OFFSET/LIMIT trick - we will need to read all these leaf nodes pages just to skip them
+   * so, in order to remove this overhead for random row selection - we creating an index with just single column used
+   * in this case B-Tree leafs will be full of rowids and the overhead for page reads will be very small
+  */
+  zSql = sqlite3MPrintf(
+      db,
+      "CREATE INDEX IF NOT EXISTS \"%w\".%s_shadow_idx ON %s_shadow (%s)",
+      zDbSName,
+      zIdxName,
+      zIdxName,
+      zRowidColumnName
+  );
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  sqlite3DbFree(db, zSql);
   return rc;
 }
 
@@ -212082,8 +212134,8 @@ static int diskAnnSelectRandomShadowRow(const DiskAnnIndex *pIndex, u64 *pRowid)
 
   zSql = sqlite3MPrintf(
     pIndex->db,
-    "SELECT rowid FROM \"%w\".%s LIMIT 1 OFFSET ABS(RANDOM()) %% MAX((SELECT COUNT(*) FROM %s), 1)",
-    pIndex->zDbSName, pIndex->zShadow, pIndex->zShadow
+    "SELECT rowid FROM \"%w\".%s LIMIT 1 OFFSET ABS(RANDOM()) %% MAX((SELECT COUNT(*) FROM \"%w\".%s), 1)",
+    pIndex->zDbSName, pIndex->zShadow, pIndex->zDbSName, pIndex->zShadow
   );
   if( zSql == NULL ){
     rc = SQLITE_NOMEM_BKPT;
@@ -213658,11 +213710,6 @@ int vectorF64ParseSqliteBlob(
 ** VectorIdxParams utilities
 ****************************************************************************/
 
-// VACUUM creates tables and indices first and only then populate data
-// we need to ignore inserts from 'INSERT INTO vacuum.t SELECT * FROM t' statements because
-// all shadow tables will be populated by VACUUM process during regular process of table copy
-#define IsVacuum(db) ((db->mDbFlags&DBFLAG_Vacuum)!=0)
-
 void vectorIdxParamsInit(VectorIdxParams *pParams, u8 *pBinBuf, int nBinSize) {
   assert( nBinSize <= VECTOR_INDEX_PARAMS_BUF_SIZE );
 
@@ -214381,10 +214428,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
   // this is done to prevent unrecoverable situations where index were dropped but index parameters deletion failed and second attempt will fail on first step
   int rcIdx, rcParams;
 
-  if( IsVacuum(db) ){
-    return SQLITE_OK;
-  }
-
   assert( zDbSName != NULL );
 
   rcIdx = diskAnnDropIndex(db, zDbSName, zIdxName);
@@ -214395,10 +214438,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
 int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
   assert( zDbSName != NULL );
 
-  if( IsVacuum(db) ){
-    return SQLITE_OK;
-  }
-
   return diskAnnClearIndex(db, zDbSName, zIdxName);
 }
 
@@ -214408,7 +214447,7 @@ int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
  * this made intentionally in order to natively support upload of SQLite dumps
  *
  * dump populates tables first and create indices after
- * so we must omit them because shadow tables already filled
+ * so we must omit index refill setp because shadow tables already filled
  *
  * 1. in case of any error                                        :-1 returned (and pParse errMsg is populated with some error message)
  * 2. if vector index must not be created                         : 0 returned
@@ -214426,10 +214465,6 @@ int vectorIndexCreate(Parse *pParse, const Index *pIdx, const char *zDbSName, co
   int hasLibsqlVectorIdxFn = 0, hasCollation = 0;
   const char *pzErrMsg;
 
-  if( IsVacuum(pParse->db) ){
-    return CREATE_IGNORE;
-  }
-
   assert( zDbSName != NULL );
 
   sqlite3 *db = pParse->db;
@@ -214488,11 +214523,6 @@ int vectorIndexCreate(Parse *pParse, const Index *pIdx, const char *zDbSName, co
     sqlite3ErrorMsg(pParse, "vector index: must contain exactly one column wrapped into the " VECTOR_INDEX_MARKER_FUNCTION " function");
     return CREATE_FAIL;
   }
-  // we are able to support this but I doubt this works for now - more polishing required to make this work
-  if( pIdx->pPartIdxWhere != NULL ) {
-    sqlite3ErrorMsg(pParse, "vector index: where condition is forbidden");
-    return CREATE_FAIL;
-  }
 
   pArgsList = pIdx->aColExpr->a[0].pExpr->x.pList;
   pListItem = pArgsList->a;
@@ -214582,7 +214612,6 @@ int vectorIndexSearch(
   VectorIdxParams idxParams;
   vectorIdxParamsInit(&idxParams, NULL, 0);
 
-  assert( !IsVacuum(db) );
   assert( zDbSName != NULL );
 
   if( argc != 3 ){
@@ -214667,10 +214696,6 @@ int vectorIndexInsert(
   int rc;
   VectorInRow vectorInRow;
 
-  if( IsVacuum(pCur->db) ){
-    return SQLITE_OK;
-  }
-
   rc = vectorInRowAlloc(pCur->db, pRecord, &vectorInRow, pzErrMsg);
   if( rc != SQLITE_OK ){
     return rc;
@@ -214690,10 +214715,6 @@ int vectorIndexDelete(
 ){
   VectorInRow payload;
 
-  if( IsVacuum(pCur->db) ){
-    return SQLITE_OK;
-  }
-
   payload.pVector = NULL;
   payload.nKeys = r->nField - 1;
   payload.pKeyValues = r->aMem + 1;
@@ -259749,40 +259770,46 @@ static int fts5TriCreate(
   Fts5Tokenizer **ppOut
 ){
   int rc = SQLITE_OK;
-  TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
-  UNUSED_PARAM(pUnused);
-  if( pNew==0 ){
-    rc = SQLITE_NOMEM;
+  TrigramTokenizer *pNew = 0;
+
+  if( nArg%2 ){
+    rc = SQLITE_ERROR;
   }else{
-    int i;
-    pNew->bFold = 1;
-    pNew->iFoldParam = 0;
-    for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
-      const char *zArg = azArg[i+1];
-      if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
-        if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
-          rc = SQLITE_ERROR;
+    pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
+    UNUSED_PARAM(pUnused);
+    if( pNew==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      int i;
+      pNew->bFold = 1;
+      pNew->iFoldParam = 0;
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+        const char *zArg = azArg[i+1];
+        if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
+          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+            rc = SQLITE_ERROR;
+          }else{
+            pNew->bFold = (zArg[0]=='0');
+          }
+        }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+            rc = SQLITE_ERROR;
+          }else{
+            pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
+          }
         }else{
-          pNew->bFold = (zArg[0]=='0');
-        }
-      }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-        if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
           rc = SQLITE_ERROR;
-        }else{
-          pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
         }
-      }else{
-        rc = SQLITE_ERROR;
       }
-    }
 
-    if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
-      rc = SQLITE_ERROR;
-    }
+      if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
+        rc = SQLITE_ERROR;
+      }
 
-    if( rc!=SQLITE_OK ){
-      fts5TriDelete((Fts5Tokenizer*)pNew);
-      pNew = 0;
+      if( rc!=SQLITE_OK ){
+        fts5TriDelete((Fts5Tokenizer*)pNew);
+        pNew = 0;
+      }
     }
   }
   *ppOut = (Fts5Tokenizer*)pNew;