diff --git a/.changeset/rare-doors-repeat.md b/.changeset/rare-doors-repeat.md new file mode 100644 index 000000000..ae0cfce6b --- /dev/null +++ b/.changeset/rare-doors-repeat.md @@ -0,0 +1,9 @@ +--- +'@powersync/common': minor +'@powersync/node': minor +'@powersync/capacitor': minor +'@powersync/react-native': minor +'@powersync/web': minor +--- + +Improve raw tables by making `put` and `delete` statements optional if a local name is given. diff --git a/packages/common/src/db/schema/RawTable.ts b/packages/common/src/db/schema/RawTable.ts index bd9a48488..7de8065b6 100644 --- a/packages/common/src/db/schema/RawTable.ts +++ b/packages/common/src/db/schema/RawTable.ts @@ -1,8 +1,23 @@ +import { TableOrRawTableOptions } from './Table.js'; + /** - * A pending variant of a {@link RawTable} that doesn't have a name (because it would be inferred when creating the - * schema). + * Instructs PowerSync to sync data into a "raw" table. + * + * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow + * using client-side table and column constraints. + * + * To collect local writes to raw tables with PowerSync, custom triggers are required. See + * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on + * using raw tables. + * + * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client. + * + * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or + * stability guarantees. */ -export type RawTableType = { +export type RawTableType = RawTableTypeWithStatements | InferredRawTableType; + +interface RawTableTypeWithStatements { /** * The statement to run when PowerSync detects that a row needs to be inserted or updated. */ @@ -11,7 +26,32 @@ export type RawTableType = { * The statement to run when PowerSync detects that a row needs to be deleted. */ delete: PendingStatement; -}; + + /** + * An optional statement to run when `disconnectAndClear()` is called on a PowerSync database. + */ + clear?: string; +} + +interface InferredRawTableType extends Partial, TableOrRawTableOptions { + /** + * The actual name of the raw table in the local schema. + * + * Unlike {@link RawTable.name}, which describes the name of synced tables to match, this reflects the SQLite table + * name. This is used to infer {@link RawTableType.put} and {@link RawTableType.delete} statements for the sync + * client. It can also be used to auto-generate triggers forwarding writes on raw tables into the CRUD upload queue + * (using the `powersync_create_raw_table_crud_trigger` SQL function). + */ + tableName: string; + + /** + * An optional filter of columns that should be synced. + * + * By default, all columns in a raw table are considered for sync. If a filter is specified, PowerSync treats + * unmatched columns as local-only and will not attempt to sync them. + */ + syncedColumns?: string[]; +} /** * A parameter to use as part of {@link PendingStatement}. @@ -21,8 +61,10 @@ export type RawTableType = { * * For insert and replace operations, the values of columns in the table are available as parameters through * `{Column: 'name'}`. + * The `"Rest"` parameter gets resolved to a JSON object covering all values from the synced row that haven't been + * covered by a `Column` parameter. */ -export type PendingStatementParameter = 'Id' | { Column: string }; +export type PendingStatementParameter = 'Id' | { Column: string } | 'Rest'; /** * A statement that the PowerSync client should use to insert or delete data into a table managed by the user. @@ -33,35 +75,16 @@ export type PendingStatement = { }; /** - * Instructs PowerSync to sync data into a "raw" table. - * - * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow - * using client-side table and column constraints. - * - * To collect local writes to raw tables with PowerSync, custom triggers are required. See - * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on - * using raw tables. - * - * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client. - * - * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or - * stability guarantees. + * @internal */ -export class RawTable implements RawTableType { +export type RawTable = T & { /** * The name of the table. * - * This does not have to match the actual table name in the schema - {@link put} and {@link delete} are free to use - * another table. Instead, this name is used by the sync client to recognize that operations on this table (as it - * appears in the source / backend database) are to be handled specially. + * This does not have to match the actual table name in the schema - {@link RawTableType.put} and + * {@link RawTableType.delete} are free to use another table. Instead, this name is used by the sync client to + * recognize that operations on this table (as it appears in the source / backend database) are to be handled + * specially. */ name: string; - put: PendingStatement; - delete: PendingStatement; - - constructor(name: string, type: RawTableType) { - this.name = name; - this.put = type.put; - this.delete = type.delete; - } -} +}; diff --git a/packages/common/src/db/schema/Schema.ts b/packages/common/src/db/schema/Schema.ts index a5f69d8f8..56c8d0cbe 100644 --- a/packages/common/src/db/schema/Schema.ts +++ b/packages/common/src/db/schema/Schema.ts @@ -1,3 +1,4 @@ +import { encodeTableOptions } from './internal.js'; import { RawTable, RawTableType } from './RawTable.js'; import { RowType, Table } from './Table.js'; @@ -57,7 +58,7 @@ export class Schema { */ withRawTables(tables: Record) { for (const [name, rawTableDefinition] of Object.entries(tables)) { - this.rawTables.push(new RawTable(name, rawTableDefinition)); + this.rawTables.push({ name, ...rawTableDefinition }); } } @@ -70,7 +71,30 @@ export class Schema { toJSON() { return { tables: this.tables.map((t) => t.toJSON()), - raw_tables: this.rawTables + raw_tables: this.rawTables.map(Schema.rawTableToJson) }; } + + /** + * Returns a representation of the raw table that is understood by the PowerSync SQLite core extension. + * + * The output of this can be passed through `JSON.serialize` and then used in `powersync_create_raw_table_crud_trigger` + * to define triggers for this table. + */ + static rawTableToJson(table: RawTable): unknown { + const serialized: any = { + name: table.name, + put: table.put, + delete: table.delete, + clear: table.clear + }; + if ('tableName' in table) { + // We have schema options + serialized.table_name = table.tableName; + serialized.synced_columns = table.syncedColumns; + Object.assign(serialized, encodeTableOptions(table)); + } + + return serialized; + } } diff --git a/packages/common/src/db/schema/Table.ts b/packages/common/src/db/schema/Table.ts index ae729681d..2d1221064 100644 --- a/packages/common/src/db/schema/Table.ts +++ b/packages/common/src/db/schema/Table.ts @@ -8,17 +8,24 @@ import { } from './Column.js'; import { Index } from './Index.js'; import { IndexedColumn } from './IndexedColumn.js'; +import { encodeTableOptions } from './internal.js'; import { TableV2 } from './TableV2.js'; -interface SharedTableOptions { +/** + * Options that apply both to JSON-based tables and raw tables. + */ +export interface TableOrRawTableOptions { localOnly?: boolean; insertOnly?: boolean; - viewName?: string; trackPrevious?: boolean | TrackPreviousOptions; trackMetadata?: boolean; ignoreEmptyUpdates?: boolean; } +interface SharedTableOptions extends TableOrRawTableOptions { + viewName?: string; +} + /** Whether to include previous column values when PowerSync tracks local changes. * * Including old values may be helpful for some backend connector implementations, which is @@ -341,19 +348,12 @@ export class Table { } toJSON() { - const trackPrevious = this.trackPrevious; - return { name: this.name, view_name: this.viewName, - local_only: this.localOnly, - insert_only: this.insertOnly, - include_old: trackPrevious && ((trackPrevious as any).columns ?? true), - include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true, - include_metadata: this.trackMetadata, - ignore_empty_update: this.ignoreEmptyUpdates, columns: this.columns.map((c) => c.toJSON()), - indexes: this.indexes.map((e) => e.toJSON(this)) + indexes: this.indexes.map((e) => e.toJSON(this)), + ...encodeTableOptions(this) }; } } diff --git a/packages/common/src/db/schema/internal.ts b/packages/common/src/db/schema/internal.ts new file mode 100644 index 000000000..b6a28fa18 --- /dev/null +++ b/packages/common/src/db/schema/internal.ts @@ -0,0 +1,17 @@ +import { TableOrRawTableOptions } from './Table.js'; + +/** + * @internal Not exported from `index.ts`. + */ +export function encodeTableOptions(options: TableOrRawTableOptions) { + const trackPrevious = options.trackPrevious; + + return { + local_only: options.localOnly, + insert_only: options.insertOnly, + include_old: trackPrevious && ((trackPrevious as any).columns ?? true), + include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true, + include_metadata: options.trackMetadata, + ignore_empty_update: options.ignoreEmptyUpdates + }; +} diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index be17cf695..d7cc90d9e 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -39,7 +39,7 @@ export * from './db/DBAdapter.js'; export * from './db/schema/Column.js'; export * from './db/schema/Index.js'; export * from './db/schema/IndexedColumn.js'; -export * from './db/schema/RawTable.js'; +export { RawTableType, PendingStatementParameter, PendingStatement } from './db/schema/RawTable.js'; export * from './db/schema/Schema.js'; export * from './db/schema/Table.js'; export * from './db/schema/TableV2.js'; diff --git a/packages/node/tests/PowerSyncDatabase.test.ts b/packages/node/tests/PowerSyncDatabase.test.ts index e3660841a..d5cbd21eb 100644 --- a/packages/node/tests/PowerSyncDatabase.test.ts +++ b/packages/node/tests/PowerSyncDatabase.test.ts @@ -1,7 +1,7 @@ import * as path from 'node:path'; import { Worker } from 'node:worker_threads'; -import { LockContext } from '@powersync/common'; +import { LockContext, Schema } from '@powersync/common'; import { randomUUID } from 'node:crypto'; import { expect, test, vi } from 'vitest'; import { CrudEntry, CrudTransaction, PowerSyncDatabase } from '../lib'; @@ -227,3 +227,21 @@ tempDirectoryTest.skipIf(process.versions.node < '22.5.0')( } } ); + +databaseTest('clear raw tables', async ({ database }) => { + await database.init(); + const schema = new Schema({}); + schema.withRawTables({ + users: { + table_name: 'lists', + clear: 'DELETE FROM lists' + } + }); + await database.updateSchema(schema); + await database.execute('CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT)'); + await database.execute('INSERT INTO lists (id, name) VALUES (uuid(), ?)', ['list']); + + expect(await database.getAll('SELECT * FROM lists')).toHaveLength(1); + await database.disconnectAndClear(); + expect(await database.getAll('SELECT * FROM lists')).toHaveLength(0); +}); diff --git a/packages/node/tests/crud.test.ts b/packages/node/tests/crud.test.ts index 5ab66196c..ff8c9bea3 100644 --- a/packages/node/tests/crud.test.ts +++ b/packages/node/tests/crud.test.ts @@ -1,6 +1,7 @@ -import { expect } from 'vitest'; -import { column, Schema, Table } from '@powersync/common'; +import { expect, describe } from 'vitest'; +import { column, Schema, Table, RawTable } from '@powersync/common'; import { databaseTest } from './utils'; +import { PowerSyncDatabase } from '../src'; databaseTest('include metadata', async ({ database }) => { await database.init(); @@ -103,3 +104,57 @@ databaseTest('ignore empty update', async ({ database }) => { const batch = await database.getNextCrudTransaction(); expect(batch).toBeNull(); }); + +describe('raw table', () => { + async function createTrigger(db: PowerSyncDatabase, table: RawTable, write: string) { + await db.execute('SELECT powersync_create_raw_table_crud_trigger(?, ?, ?)', [ + JSON.stringify(Schema.rawTableToJson(table)), + `users_${write}`, + write + ]); + } + + databaseTest('inferred crud trigger', async ({ database }) => { + const table: RawTable = { name: 'sync_name', tableName: 'users' }; + await database.execute('CREATE TABLE users (id TEXT, name TEXT);'); + await createTrigger(database, table, 'INSERT'); + + await database.execute('INSERT INTO users (id, name) VALUES (?, ?);', ['id', 'user']); + const tx = await database.getNextCrudTransaction()!!; + expect(tx.crud).toHaveLength(1); + const write = tx.crud[0]; + expect(write.op).toStrictEqual('PUT'); + expect(write.table).toStrictEqual('sync_name'); + expect(write.id).toStrictEqual('id'); + expect(write.opData).toStrictEqual({ + name: 'user' + }); + }); + + databaseTest('with options', async ({ database }) => { + const table: RawTable = { + name: 'sync_name', + tableName: 'users', + syncedColumns: ['name'], + ignoreEmptyUpdates: true, + trackPrevious: true + }; + await database.execute('CREATE TABLE users (id TEXT, name TEXT, local TEXT);'); + await database.execute('INSERT INTO users (id, name, local) VALUES (?, ?, ?);', ['id', 'name', 'local']); + await createTrigger(database, table, 'UPDATE'); + + await database.execute('UPDATE users SET name = ?, local = ?;', ['updated_name', 'updated_local']); + // This should not generate a CRUD entry because the only synced column is not affected. + await database.execute('UPDATE users SET name = ?, local = ?;', ['name', 'updated_local_2']); + + const tx = await database.getNextCrudTransaction()!!; + expect(tx.crud).toHaveLength(1); + const write = tx.crud[0]; + expect(write.op).toStrictEqual('PATCH'); + expect(write.id).toStrictEqual('id'); + expect(write.opData).toStrictEqual({ name: 'updated_name' }); + expect(write.previousValues).toStrictEqual({ + name: 'name' + }); + }); +}); diff --git a/packages/node/tests/sync.test.ts b/packages/node/tests/sync.test.ts index 3461b6fdb..7d503db3d 100644 --- a/packages/node/tests/sync.test.ts +++ b/packages/node/tests/sync.test.ts @@ -725,13 +725,83 @@ function defineSyncTests(impl: SyncClientImplementation) { }); if (impl == SyncClientImplementation.RUST) { - mockSyncServiceTest('raw tables', async ({ syncService }) => { + mockSyncServiceTest('raw tables with inferred statements', async ({ syncService }) => { + const customSchema = new Schema({}); + customSchema.withRawTables({ + lists: { + tableName: 'lists' + } + }); + + const powersync = await syncService.createDatabase({ schema: customSchema }); + await powersync.execute('CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT);'); + + const query = powersync.watchWithAsyncGenerator('SELECT * FROM lists')[Symbol.asyncIterator](); + expect((await query.next()).value.rows._array).toStrictEqual([]); + + powersync.connect(new TestConnector(), options); + await vi.waitFor(() => expect(syncService.connectedListeners).toHaveLength(1)); + + syncService.pushLine({ + checkpoint: { + last_op_id: '1', + buckets: [bucket('a', 1)] + } + }); + syncService.pushLine({ + data: { + bucket: 'a', + data: [ + { + checksum: 0, + op_id: '1', + op: 'PUT', + object_id: 'my_list', + object_type: 'lists', + data: '{"name": "custom list"}' + } + ] + } + }); + syncService.pushLine({ checkpoint_complete: { last_op_id: '1' } }); + await powersync.waitForFirstSync(); + + expect((await query.next()).value.rows._array).toStrictEqual([{ id: 'my_list', name: 'custom list' }]); + + syncService.pushLine({ + checkpoint: { + last_op_id: '2', + buckets: [bucket('a', 2)] + } + }); + await vi.waitFor(() => powersync.currentStatus.dataFlowStatus.downloading == true); + syncService.pushLine({ + data: { + bucket: 'a', + data: [ + { + checksum: 0, + op_id: '2', + op: 'REMOVE', + object_id: 'my_list', + object_type: 'lists' + } + ] + } + }); + syncService.pushLine({ checkpoint_complete: { last_op_id: '2' } }); + await vi.waitFor(() => powersync.currentStatus.dataFlowStatus.downloading == false); + + expect((await query.next()).value.rows._array).toStrictEqual([]); + }); + + mockSyncServiceTest('raw tables with explicit statements', async ({ syncService }) => { const customSchema = new Schema({}); customSchema.withRawTables({ lists: { put: { - sql: 'INSERT OR REPLACE INTO lists (id, name) VALUES (?, ?)', - params: ['Id', { Column: 'name' }] + sql: 'INSERT OR REPLACE INTO lists (id, name, _rest) VALUES (?, ?, ?)', + params: ['Id', { Column: 'name' }, 'Rest'] }, delete: { sql: 'DELETE FROM lists WHERE id = ?', @@ -741,7 +811,7 @@ function defineSyncTests(impl: SyncClientImplementation) { }); const powersync = await syncService.createDatabase({ schema: customSchema }); - await powersync.execute('CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT);'); + await powersync.execute('CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT, _rest TEXT);'); const query = powersync.watchWithAsyncGenerator('SELECT * FROM lists')[Symbol.asyncIterator](); expect((await query.next()).value.rows._array).toStrictEqual([]); @@ -765,7 +835,7 @@ function defineSyncTests(impl: SyncClientImplementation) { op: 'PUT', object_id: 'my_list', object_type: 'lists', - data: '{"name": "custom list"}' + data: '{"name": "custom list", "additional": "foo"}' } ] } @@ -773,7 +843,9 @@ function defineSyncTests(impl: SyncClientImplementation) { syncService.pushLine({ checkpoint_complete: { last_op_id: '1' } }); await powersync.waitForFirstSync(); - expect((await query.next()).value.rows._array).toStrictEqual([{ id: 'my_list', name: 'custom list' }]); + expect((await query.next()).value.rows._array).toStrictEqual([ + { id: 'my_list', name: 'custom list', _rest: '{"additional":"foo"}' } + ]); syncService.pushLine({ checkpoint: {