From 6981d4ce8136efda4d2a959416b0030a5b1b841e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Feb 2026 10:13:05 +0100 Subject: [PATCH 1/7] Store precompiled sync plans in storage --- .../src/storage/MongoBucketStorage.ts | 1 + .../MongoPersistedSyncRulesContent.ts | 1 + .../src/storage/implementation/models.ts | 3 +- .../scripts/1771491856000-sync-plan.ts | 23 +++++++++ .../storage/PostgresBucketStorageFactory.ts | 13 +++-- .../PostgresPersistedSyncRulesContent.ts | 8 +-- .../src/types/models/SyncRules.ts | 13 ++++- .../src/routes/endpoints/admin.ts | 3 +- .../src/storage/BucketStorageFactory.ts | 51 +++++++++++++++++-- .../src/storage/PersistedSyncRulesContent.ts | 37 +++++++++++++- packages/sync-rules/src/SyncConfig.ts | 4 +- 11 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 modules/module-postgres-storage/src/migrations/scripts/1771491856000-sync-plan.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 0f11064af..d4ac9f067 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -179,6 +179,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { _id: id, storage_version: storageVersion, content: options.config.yaml, + serialized_plan: options.config.plan, last_checkpoint: null, last_checkpoint_lsn: null, no_checkpoint_before: null, diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index f1b1bd47f..c946868b6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -15,6 +15,7 @@ export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesCo super({ id: doc._id, sync_rules_content: doc.content, + compiled_plan: doc.serialized_plan ?? null, last_checkpoint_lsn: doc.last_checkpoint_lsn, // Handle legacy values slot_name: doc.slot_name ?? `powersync_${doc._id}`, diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index a3c2110df..bb95e4c6c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -1,4 +1,4 @@ -import { InternalOpId, storage } from '@powersync/service-core'; +import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core'; import { SqliteJsonValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { event_types } from '@powersync/service-types'; @@ -199,6 +199,7 @@ export interface SyncRuleDocument { last_fatal_error_ts: Date | null; content: string; + serialized_plan?: SerializedSyncPlan | null; lock?: { id: string; diff --git a/modules/module-postgres-storage/src/migrations/scripts/1771491856000-sync-plan.ts b/modules/module-postgres-storage/src/migrations/scripts/1771491856000-sync-plan.ts new file mode 100644 index 000000000..7d4fee658 --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/scripts/1771491856000-sync-plan.ts @@ -0,0 +1,23 @@ +import { migrations } from '@powersync/service-core'; +import { openMigrationDB } from '../migration-utils.js'; + +export const up: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + await client.transaction(async (db) => { + await db.sql` + ALTER TABLE sync_rules + ADD COLUMN sync_plan JSON; + `.execute(); + }); +}; + +export const down: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + await client.sql`ALTER TABLE sync_rules DROP COLUMN sync_plan`.execute(); +}; diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 176781298..43441ca80 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -1,6 +1,5 @@ -import { GetIntanceOptions, storage, SyncRulesBucketStorage, UpdateSyncRulesOptions } from '@powersync/service-core'; +import { GetIntanceOptions, storage, SyncRulesBucketStorage } from '@powersync/service-core'; import * as pg_wire from '@powersync/service-jpgwire'; -import * as sync_rules from '@powersync/service-sync-rules'; import crypto from 'crypto'; import * as uuid from 'uuid'; @@ -159,7 +158,14 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { nextval('sync_rules_id_sequence') AS id ) INSERT INTO - sync_rules (id, content, state, slot_name, storage_version) + sync_rules ( + id, + content, + sync_plan, + state, + slot_name, + storage_version + ) VALUES ( ( @@ -169,6 +175,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { next_id ), ${{ type: 'varchar', value: options.config.yaml }}, + ${{ type: 'json', value: options.config.plan }}, ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}, CONCAT( ${{ type: 'varchar', value: this.slot_name_prefix }}, diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts index 79bc12e70..476cf929b 100644 --- a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -1,13 +1,6 @@ import * as lib_postgres from '@powersync/lib-service-postgres'; import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; -import { - CompatibilityOption, - DEFAULT_HYDRATION_STATE, - HydrationState, - SqlSyncRules, - versionedHydrationState -} from '@powersync/service-sync-rules'; import { models } from '../../types/types.js'; export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRulesContent { @@ -20,6 +13,7 @@ export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRule super({ id: Number(row.id), sync_rules_content: row.content, + compiled_plan: row.sync_plan, last_checkpoint_lsn: row.last_checkpoint_lsn, slot_name: row.slot_name, last_fatal_error: row.last_fatal_error, diff --git a/modules/module-postgres-storage/src/types/models/SyncRules.ts b/modules/module-postgres-storage/src/types/models/SyncRules.ts index 8f94e45e9..81ac99070 100644 --- a/modules/module-postgres-storage/src/types/models/SyncRules.ts +++ b/modules/module-postgres-storage/src/types/models/SyncRules.ts @@ -48,7 +48,18 @@ export const SyncRules = t.object({ last_fatal_error: t.Null.or(t.string), keepalive_op: t.Null.or(bigint), storage_version: t.Null.or(pgwire_number).optional(), - content: t.string + content: t.string, + sync_plan: t.Null.or( + t.object({ + plan: t.any, + compatibility: t.object({ + edition: t.number, + overrides: t.record(t.boolean), + maxTimeValuePrecision: t.number.optional() + }), + eventDescriptors: t.record(t.array(t.string)) + }) + ) }); export type SyncRules = t.Encoded; diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 51461a557..59eca123a 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -205,7 +205,8 @@ export const validate = routeDefinition({ active: false, last_checkpoint_lsn: '', storageVersion: storage.LEGACY_STORAGE_VERSION, - sync_rules_content: content + sync_rules_content: content, + compiled_plan: null }); const connectionStatus = await apiHandler.getConnectionStatus(); diff --git a/packages/service-core/src/storage/BucketStorageFactory.ts b/packages/service-core/src/storage/BucketStorageFactory.ts index 1b69355af..2b2cc59d9 100644 --- a/packages/service-core/src/storage/BucketStorageFactory.ts +++ b/packages/service-core/src/storage/BucketStorageFactory.ts @@ -4,7 +4,13 @@ import { ReplicationEventPayload } from './ReplicationEventPayload.js'; import { ReplicationLock } from './ReplicationLock.js'; import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js'; import { ReportStorage } from './ReportStorage.js'; -import { SqlSyncRules, SyncConfig } from '@powersync/service-sync-rules'; +import { + PrecompiledSyncConfig, + SerializedCompatibilityContext, + serializeSyncPlan, + SqlSyncRules, + SyncConfig +} from '@powersync/service-sync-rules'; /** * Represents a configured storage provider. @@ -148,12 +154,32 @@ export interface StorageMetrics { export interface UpdateSyncRulesOptions { config: { yaml: string; - // TODO: Add serialized sync plan if available + /** + * The serialized sync plan for the sync configuration, or `null` for configurations not using the sync stream + * compiler. + */ + plan: SerializedSyncPlan | null; }; lock?: boolean; storageVersion?: number; } +export interface SerializedSyncPlan { + /** + * The serialized plan, from {@link serializeSyncPlan}. + */ + plan: unknown; + compatibility: SerializedCompatibilityContext; + /** + * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler + * yet. + * + * We might revisit that in the future, but for now we store SQL text of their definitions here to be able to restore + * them. + */ + eventDescriptors: Record; +} + export function updateSyncRulesFromYaml( content: string, options?: Omit & { validate?: boolean } @@ -168,8 +194,25 @@ export function updateSyncRulesFromYaml( return updateSyncRulesFromConfig(config, options); } -export function updateSyncRulesFromConfig(parsed: SyncConfig, options?: Omit) { - return { config: { yaml: parsed.content }, ...options }; +export function updateSyncRulesFromConfig( + parsed: SyncConfig, + options?: Omit +): UpdateSyncRulesOptions { + let plan: SerializedSyncPlan | null = null; + if (parsed instanceof PrecompiledSyncConfig) { + const eventDescriptors: Record = {}; + for (const event of parsed.eventDescriptors) { + eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql); + } + + plan = { + compatibility: parsed.compatibility.serialize(), + plan: serializeSyncPlan(parsed.plan), + eventDescriptors + }; + } + + return { config: { yaml: parsed.content, plan }, ...options }; } export interface GetIntanceOptions { diff --git a/packages/service-core/src/storage/PersistedSyncRulesContent.ts b/packages/service-core/src/storage/PersistedSyncRulesContent.ts index da8bef840..f9bf76622 100644 --- a/packages/service-core/src/storage/PersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/PersistedSyncRulesContent.ts @@ -1,16 +1,22 @@ import { + CompatibilityContext, CompatibilityOption, DEFAULT_HYDRATION_STATE, + deserializeSyncPlan, HydratedSyncRules, HydrationState, + javaScriptExpressionEngine, + PrecompiledSyncConfig, + SqlEventDescriptor, SqlSyncRules, + SyncConfig, SyncConfigWithErrors, versionedHydrationState } from '@powersync/service-sync-rules'; import { ReplicationLock } from './ReplicationLock.js'; import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js'; import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; -import { UpdateSyncRulesOptions } from './BucketStorageFactory.js'; +import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js'; export interface ParseSyncRulesOptions { defaultSchema: string; @@ -19,6 +25,7 @@ export interface ParseSyncRulesOptions { export interface PersistedSyncRulesContentData { readonly id: number; readonly sync_rules_content: string; + readonly compiled_plan: SerializedSyncPlan | null; readonly slot_name: string; /** * True if this is the "active" copy of the sync rules. @@ -37,6 +44,7 @@ export interface PersistedSyncRulesContentData { export abstract class PersistedSyncRulesContent implements PersistedSyncRulesContentData { readonly id!: number; readonly sync_rules_content!: string; + readonly compiled_plan!: SerializedSyncPlan | null; readonly slot_name!: string; readonly active!: boolean; readonly storageVersion!: number; @@ -72,6 +80,31 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon parsed(options: ParseSyncRulesOptions): PersistedSyncRules { let hydrationState: HydrationState; + + // Do we have a compiled sync plan? If so, restore from there instead of parsing everything again. + let config: SyncConfigWithErrors; + if (this.compiled_plan != null) { + const plan = deserializeSyncPlan(this.compiled_plan.plan); + const compatibility = CompatibilityContext.deserialize(this.compiled_plan.compatibility); + + const precompiled = new PrecompiledSyncConfig(plan, { + defaultSchema: options.defaultSchema, + engine: javaScriptExpressionEngine(compatibility), + sourceText: this.sync_rules_content + }); + + for (const [name, queries] of Object.entries(this.compiled_plan.eventDescriptors)) { + const descriptor = new SqlEventDescriptor(name, compatibility); + for (const query of queries) { + descriptor.addSourceQuery(query, options); + } + } + + config = { config: precompiled, errors: [] }; + } else { + config = SqlSyncRules.fromYaml(this.sync_rules_content, options); + } + const syncRules = SqlSyncRules.fromYaml(this.sync_rules_content, options); const storageConfig = this.getStorageConfig(); if ( @@ -95,7 +128,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon asUpdateOptions(options?: Omit): UpdateSyncRulesOptions { return { - config: { yaml: this.sync_rules_content }, + config: { yaml: this.sync_rules_content, plan: this.compiled_plan }, ...options }; } diff --git a/packages/sync-rules/src/SyncConfig.ts b/packages/sync-rules/src/SyncConfig.ts index 19f42722f..14598d5c2 100644 --- a/packages/sync-rules/src/SyncConfig.ts +++ b/packages/sync-rules/src/SyncConfig.ts @@ -1,11 +1,9 @@ import { BucketDataSource, BucketSource, CreateSourceParams, ParameterIndexLookupCreator } from './BucketSource.js'; -import { CompatibilityContext, CompatibilityOption } from './compatibility.js'; +import { CompatibilityContext } from './compatibility.js'; import { YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; import { HydratedSyncRules } from './HydratedSyncRules.js'; -import { DEFAULT_HYDRATION_STATE } from './HydrationState.js'; import { SourceTableInterface } from './SourceTableInterface.js'; -import { SyncPlan } from './sync_plan/plan.js'; import { TablePattern } from './TablePattern.js'; import { SqliteInputValue, SqliteRow, SqliteValue } from './types.js'; import { applyRowContext } from './utils.js'; From a908f943004c60540af7be199573212b37d4a86f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Feb 2026 10:15:03 +0100 Subject: [PATCH 2/7] Mark sync plans as stable! --- .../sync-rules/src/sync_plan/serialize.ts | 7 ++- .../__snapshots__/advanced.test.ts.snap | 28 +++++------ .../__snapshots__/compatibility.test.ts.snap | 48 +++++++++---------- .../compiler/__snapshots__/cte.test.ts.snap | 4 +- 4 files changed, 43 insertions(+), 44 deletions(-) diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 994b15ab6..616fdae93 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -154,7 +154,7 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlanUnstable { } return { - version: 'unstable', // TODO: Mature to 1 before storing in bucket storage + version: 1, dataSources: serializeDataSources(), buckets: plan.buckets.map((bkt, index) => { bucketIndex.set(bkt, index); @@ -173,8 +173,7 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlanUnstable { } export function deserializeSyncPlan(serialized: unknown): SyncPlan { - // TODO: Mature to version 1 - if ((serialized as SerializedSyncPlanUnstable).version != 'unstable') { + if ((serialized as SerializedSyncPlanUnstable).version != 1) { throw new Error('Unknown sync plan version passed to deserializeSyncPlan()'); } @@ -309,7 +308,7 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { } interface SerializedSyncPlanUnstable { - version: 'unstable'; + version: number; dataSources: SerializedDataSource[]; buckets: SerializedBucketDataSource[]; parameterIndexes: SerializedParameterIndexLookupCreator[]; diff --git a/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap b/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap index 32a4d4669..6e6d2a4f3 100644 --- a/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap +++ b/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap @@ -66,7 +66,7 @@ exports[`new sync stream features > in array 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -243,7 +243,7 @@ exports[`new sync stream features > joins feedback > response 1 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -387,7 +387,7 @@ exports[`new sync stream features > joins feedback > response 4 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -589,7 +589,7 @@ exports[`new sync stream features > joins feedback > response 5 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -718,7 +718,7 @@ exports[`new sync stream features > joins feedback > response 8 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -895,7 +895,7 @@ exports[`new sync stream features > joins feedback > response 9 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1120,7 +1120,7 @@ exports[`new sync stream features > joins feedback > response 10 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1249,7 +1249,7 @@ exports[`new sync stream features > joins feedback > response 11 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1476,7 +1476,7 @@ exports[`new sync stream features > joins feedback > response 12 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1716,7 +1716,7 @@ exports[`new sync stream features > joins feedback > response 13 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1929,7 +1929,7 @@ exports[`new sync stream features > order-independent parameters 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2022,7 +2022,7 @@ exports[`new sync stream features > table-valued functions > partition on data 1 }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2165,7 +2165,7 @@ exports[`new sync stream features > table-valued functions > partition on parame }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2243,6 +2243,6 @@ exports[`new sync stream features > table-valued functions > static filter 1`] = }, }, ], - "version": "unstable", + "version": 1, } `; diff --git a/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap b/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap index 2ccf26c8a..48a01b4f8 100644 --- a/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap +++ b/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap @@ -186,7 +186,7 @@ exports[`old streams test > OR in subquery 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -288,7 +288,7 @@ exports[`old streams test > in > on parameter data 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -464,7 +464,7 @@ exports[`old streams test > in > on parameter data and table 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -590,7 +590,7 @@ exports[`old streams test > in > parameter and auth match on same column 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -701,7 +701,7 @@ exports[`old streams test > in > parameter value in subquery 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -830,7 +830,7 @@ exports[`old streams test > in > row value in subquery 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1034,7 +1034,7 @@ exports[`old streams test > in > two subqueries 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1189,7 +1189,7 @@ exports[`old streams test > nested subqueries 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1405,7 +1405,7 @@ exports[`old streams test > normalization > distribute and 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1534,7 +1534,7 @@ exports[`old streams test > normalization > double negation 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1716,7 +1716,7 @@ exports[`old streams test > normalization > negated and 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1870,7 +1870,7 @@ exports[`old streams test > normalization > negated or 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -1994,7 +1994,7 @@ exports[`old streams test > or > parameter match or request condition 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2122,7 +2122,7 @@ exports[`old streams test > or > parameter match or row condition 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2209,7 +2209,7 @@ exports[`old streams test > or > request condition or request condition 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2325,7 +2325,7 @@ exports[`old streams test > or > row condition or parameter condition 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2420,7 +2420,7 @@ exports[`old streams test > or > row condition or row condition 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2594,7 +2594,7 @@ exports[`old streams test > or > subquery or token parameter 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2737,7 +2737,7 @@ exports[`old streams test > overlap 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2808,7 +2808,7 @@ exports[`old streams test > row condition 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2908,7 +2908,7 @@ exports[`old streams test > row filter and stream parameter 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -2987,7 +2987,7 @@ exports[`old streams test > stream parameter 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -3116,7 +3116,7 @@ exports[`old streams test > table alias 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -3166,6 +3166,6 @@ exports[`old streams test > without filter 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; diff --git a/packages/sync-rules/test/src/compiler/__snapshots__/cte.test.ts.snap b/packages/sync-rules/test/src/compiler/__snapshots__/cte.test.ts.snap index ac34de255..eb5894d9c 100644 --- a/packages/sync-rules/test/src/compiler/__snapshots__/cte.test.ts.snap +++ b/packages/sync-rules/test/src/compiler/__snapshots__/cte.test.ts.snap @@ -142,7 +142,7 @@ exports[`common table expressions > as data source 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; @@ -319,6 +319,6 @@ exports[`common table expressions > as parameter query 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; From 91674b52bd75cbc9f7718424960a8914b8bc2595 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Feb 2026 10:21:06 +0100 Subject: [PATCH 3/7] Add changesets --- .changeset/real-houses-lie.md | 5 +++++ .changeset/thin-paws-battle.md | 7 +++++++ 2 files changed, 12 insertions(+) create mode 100644 .changeset/real-houses-lie.md create mode 100644 .changeset/thin-paws-battle.md diff --git a/.changeset/real-houses-lie.md b/.changeset/real-houses-lie.md new file mode 100644 index 000000000..952d9a394 --- /dev/null +++ b/.changeset/real-houses-lie.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-sync-rules': patch +--- + +Mark sync plans as stable. diff --git a/.changeset/thin-paws-battle.md b/.changeset/thin-paws-battle.md new file mode 100644 index 000000000..ba3c40251 --- /dev/null +++ b/.changeset/thin-paws-battle.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core': minor +--- + +Store compiled sync plans in bucket storage. From a9b5418ccb86156ad6a7403169a292605ef60a0d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Feb 2026 15:56:01 +0100 Subject: [PATCH 4/7] Parse with new compiler --- .../src/storage/PersistedSyncRulesContent.ts | 7 +++---- packages/sync-rules/src/SqlSyncRules.ts | 9 --------- packages/sync-rules/src/from_yaml.ts | 14 +++++++------- packages/sync-rules/test/src/compiler/utils.ts | 1 - 4 files changed, 10 insertions(+), 21 deletions(-) diff --git a/packages/service-core/src/storage/PersistedSyncRulesContent.ts b/packages/service-core/src/storage/PersistedSyncRulesContent.ts index f9bf76622..802298d2b 100644 --- a/packages/service-core/src/storage/PersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/PersistedSyncRulesContent.ts @@ -105,11 +105,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon config = SqlSyncRules.fromYaml(this.sync_rules_content, options); } - const syncRules = SqlSyncRules.fromYaml(this.sync_rules_content, options); const storageConfig = this.getStorageConfig(); if ( storageConfig.versionedBuckets || - syncRules.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) + config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) ) { hydrationState = versionedHydrationState(this.id); } else { @@ -119,9 +118,9 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon return { id: this.id, slot_name: this.slot_name, - sync_rules: syncRules, + sync_rules: config, hydratedSyncRules: () => { - return syncRules.config.hydrate({ hydrationState }); + return config.config.hydrate({ hydrationState }); } }; } diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 29fb6a84e..69aa0410f 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -14,14 +14,6 @@ export interface SyncRulesOptions { defaultSchema: string; throwOnError?: boolean; - - /** - * Whether to allow the option of using the new sync compiler. - * - * This is currently disabled outside of tests because the format of sync plans is still unstable and we can't support - * deployments based on it yet. Once we have a stable sync plan, this option can be removed. - */ - allowNewSyncCompiler?: boolean; } export interface RequestedStream { @@ -83,7 +75,6 @@ export class SqlSyncRules extends SyncConfig { const parser = new SyncConfigFromYaml( { throwOnError: options.throwOnError ?? true, - allowNewSyncCompiler: options.allowNewSyncCompiler ?? false, schema: options.schema, defaultSchema: options.defaultSchema }, diff --git a/packages/sync-rules/src/from_yaml.ts b/packages/sync-rules/src/from_yaml.ts index c4986dec0..124d341e8 100644 --- a/packages/sync-rules/src/from_yaml.ts +++ b/packages/sync-rules/src/from_yaml.ts @@ -82,7 +82,7 @@ export class SyncConfigFromYaml { const streamMap = parsed.get('streams') as YAMLMap | null; let result: SyncConfig; - if (useNewCompiler && this.options.allowNewSyncCompiler) { + if (useNewCompiler) { result = this.#compileSyncPlan(bucketMap, streamMap, compatibility); } else { result = this.#legacyParseBucketDefinitionsAndStreams(bucketMap, streamMap, compatibility); @@ -222,11 +222,16 @@ export class SyncConfigFromYaml { streamCompiler.finish(); } - return new PrecompiledSyncConfig(compiler.output.toSyncPlan(), { + const config = new PrecompiledSyncConfig(compiler.output.toSyncPlan(), { defaultSchema: this.options.defaultSchema, engine: javaScriptExpressionEngine(compatibility), sourceText: this.yaml }); + // We still need to store the compatibility with the precompiled sync config because it doesn't just affect how + // we compile sync streams. They also affects how the replicators encode custom types or what JSON compatibility + // options we need. + config.compatibility = compatibility; + return config; } #legacyParseBucketDefinitionsAndStreams( @@ -507,9 +512,4 @@ export interface SyncConfigFromYamlOptions { * 'public' for Postgres, default database for MongoDB/MySQL. */ readonly defaultSchema: string; - - /** - * Whether to allow the option of using the new sync compiler. - */ - readonly allowNewSyncCompiler: boolean; } diff --git a/packages/sync-rules/test/src/compiler/utils.ts b/packages/sync-rules/test/src/compiler/utils.ts index adcfd8ae9..e0b2310b3 100644 --- a/packages/sync-rules/test/src/compiler/utils.ts +++ b/packages/sync-rules/test/src/compiler/utils.ts @@ -75,7 +75,6 @@ export function yamlToSyncPlan( ): [TranslationError[], SyncPlan] { const { config, errors } = SqlSyncRules.fromYaml(source, { throwOnError: false, - allowNewSyncCompiler: true, ...options }); From c31d72d1729628b471cc30eb58a1ad667e877991 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Feb 2026 17:27:07 +0100 Subject: [PATCH 5/7] Fix reading sync plans from postgres --- .../src/types/models/SyncRules.ts | 21 ++++++++------- .../src/types/models/json.ts | 26 +++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 modules/module-postgres-storage/src/types/models/json.ts diff --git a/modules/module-postgres-storage/src/types/models/SyncRules.ts b/modules/module-postgres-storage/src/types/models/SyncRules.ts index 81ac99070..9ed422228 100644 --- a/modules/module-postgres-storage/src/types/models/SyncRules.ts +++ b/modules/module-postgres-storage/src/types/models/SyncRules.ts @@ -1,6 +1,7 @@ import { framework, storage } from '@powersync/service-core'; import * as t from 'ts-codec'; import { bigint, pgwire_number } from '../codecs.js'; +import { jsonContainerObject } from './json.js'; export const SyncRules = t.object({ id: pgwire_number, @@ -50,15 +51,17 @@ export const SyncRules = t.object({ storage_version: t.Null.or(pgwire_number).optional(), content: t.string, sync_plan: t.Null.or( - t.object({ - plan: t.any, - compatibility: t.object({ - edition: t.number, - overrides: t.record(t.boolean), - maxTimeValuePrecision: t.number.optional() - }), - eventDescriptors: t.record(t.array(t.string)) - }) + jsonContainerObject( + t.object({ + plan: t.any, + compatibility: t.object({ + edition: t.number, + overrides: t.record(t.boolean), + maxTimeValuePrecision: t.number.optional() + }), + eventDescriptors: t.record(t.array(t.string)) + }) + ) ) }); diff --git a/modules/module-postgres-storage/src/types/models/json.ts b/modules/module-postgres-storage/src/types/models/json.ts new file mode 100644 index 000000000..3716c4350 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/json.ts @@ -0,0 +1,26 @@ +import { JsonContainer } from '@powersync/service-jsonbig'; +import { Codec, codec } from 'ts-codec'; + +/** + * Wraps a codec to support {@link JsonContainer} values. + * + * Because our postgres client implementation wraps JSON objects in a {@link JsonContainer}, this intermediate layer is + * required to use JSON columns from Postgres in `ts-codec` models. + * + * Note that this serializes and deserializes values using {@link JSON}, so bigints are not supported. + */ +export function jsonContainerObject(inner: Codec): Codec { + return codec( + inner._tag, + (input) => { + return new JsonContainer(JSON.stringify(inner.encode(input))); + }, + (json) => { + if (!(json instanceof JsonContainer)) { + throw new Error('Expected JsonContainer'); + } + + return inner.decode(JSON.parse(json.data)); + } + ); +} From 098d7c42c2817eef91c36d3bfaf8f9cd3b4c6a84 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Feb 2026 17:28:52 +0100 Subject: [PATCH 6/7] Fix bad merge --- .../test/src/compiler/__snapshots__/advanced.test.ts.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap b/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap index 6e6d2a4f3..16587d7ed 100644 --- a/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap +++ b/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap @@ -1790,7 +1790,7 @@ exports[`new sync stream features > not in array 1`] = ` }, }, ], - "version": "unstable", + "version": 1, } `; From d92e1258c58e377370fa07d5a8b5fbfdaac8221f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Feb 2026 17:52:49 +0100 Subject: [PATCH 7/7] Test deploying sync streams --- .../register-data-storage-parameter-tests.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 1b9763921..63e59b475 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -608,4 +608,52 @@ bucket_definitions: expect(parsedSchema3).not.equals(parsedSchema2); expect(parsedSchema3.getSourceTables()[0].schema).equals('databasename'); }); + + test('sync streams smoke test', async () => { + await using factory = await generateStorageFactory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml(` +config: + edition: 2 + sync_config_compiler: true + +streams: + stream: + query: | + SELECT data.* FROM test AS data, test AS param + WHERE data.foo = param.bar AND param.baz = auth.user_id() + `) + ); + const bucketStorage = factory.getInstance(syncRules); + + await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await batch.save({ + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + baz: 'baz', + bar: 'bar' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await batch.commit('1/1'); + }); + + const checkpoint = await bucketStorage.getCheckpoint(); + const parameters = await checkpoint.getParameterSets([ + ScopedParameterLookup.direct( + { + lookupName: 'lookup', + queryId: '0' + }, + ['baz'] + ) + ]); + expect(parameters).toEqual([ + { + '0': 'bar' + } + ]); + }); }