Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .changeset/neat-ears-sort.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
---
'@powersync/service-sync-rules': minor
'@powersync/service-module-postgres-storage': patch
'@powersync/service-module-mongodb-storage': patch
'@powersync/service-core-tests': patch
'@powersync/service-module-postgres': patch
'@powersync/service-module-mongodb': patch
'@powersync/service-core': patch
---

Introduce `BaseSyncConfig` to represent SQL-based sync rules and precompiled sync plans.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { SqlSyncRules, HydratedSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
import { SyncConfigWithErrors, HydratedSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';

import { storage } from '@powersync/service-core';

Expand All @@ -7,14 +7,14 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules {

constructor(
public readonly id: number,
public readonly sync_rules: SqlSyncRules,
public readonly sync_rules: SyncConfigWithErrors,
public readonly checkpoint_lsn: string | null,
slot_name: string | null
) {
this.slot_name = slot_name ?? `powersync_${id}`;
}

hydratedSyncRules(): HydratedSyncRules {
return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(this.id) });
return this.sync_rules.config.hydrate({ hydrationState: versionedHydrationState(this.id) });
}
}
2 changes: 1 addition & 1 deletion modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {

async getDebugTablesInfo(
tablePatterns: sync_rules.TablePattern[],
sqlSyncRules: sync_rules.SqlSyncRules
sqlSyncRules: sync_rules.SyncConfig
): Promise<api.PatternResult[]> {
let result: api.PatternResult[] = [];

Expand Down
6 changes: 3 additions & 3 deletions modules/module-mongodb/test/src/mongo_test.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ describe('mongo data types', () => {
await setupTable(db);
await insert(collection);

const rules = SqlSyncRules.fromYaml(
const { config: rules } = SqlSyncRules.fromYaml(
`
bucket_definitions:
global:
Expand Down Expand Up @@ -457,7 +457,7 @@ bucket_definitions:
await setupTable(db);
await insert(collection);

const rules = SqlSyncRules.fromYaml(
const { config: rules } = SqlSyncRules.fromYaml(
`
bucket_definitions:
global:
Expand Down Expand Up @@ -505,7 +505,7 @@ bucket_definitions:
await setupTable(db);
await insert(collection);

const rules = SqlSyncRules.fromYaml(
const { config: rules } = SqlSyncRules.fromYaml(
`
bucket_definitions:
global:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR
slot_name: this.slot_name,
sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options),
hydratedSyncRules() {
return this.sync_rules.hydrate({
return this.sync_rules.config.hydrate({
hydrationState: versionedHydrationState(this.id)
});
}
Expand Down
4 changes: 2 additions & 2 deletions modules/module-postgres/src/replication/replication-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ export interface GetDebugTablesInfoOptions {
publicationName: string;
connectionTag: string;
tablePatterns: sync_rules.TablePattern[];
syncRules: sync_rules.SqlSyncRules;
syncRules: sync_rules.SyncConfig;
}

export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Promise<PatternResult[]> {
Expand Down Expand Up @@ -296,7 +296,7 @@ export interface GetDebugTableInfoOptions {
connectionTag: string;
tablePattern: sync_rules.TablePattern;
relationId: number | null;
syncRules: sync_rules.SqlSyncRules;
syncRules: sync_rules.SyncConfig;
}

export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Promise<service_types.TableInfo> {
Expand Down
4 changes: 2 additions & 2 deletions modules/module-postgres/test/src/validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ bucket_definitions:

const syncRules = await context.factory.updateSyncRules({ content: syncRuleContent });

const tablePatterns = syncRules.parsed({ defaultSchema: 'public' }).sync_rules.getSourceTables();
const tablePatterns = syncRules.parsed({ defaultSchema: 'public' }).sync_rules.config.getSourceTables();
const tableInfo = await getDebugTablesInfo({
db: pool,
publicationName: context.publicationName,
connectionTag: context.connectionTag,
tablePatterns: tablePatterns,
syncRules: syncRules.parsed({ defaultSchema: 'public' }).sync_rules
syncRules: syncRules.parsed({ defaultSchema: 'public' }).sync_rules.config
});
expect(tableInfo).toEqual([
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export function testRules(content: string): storage.PersistedSyncRulesContent {
sync_rules: SqlSyncRules.fromYaml(content, options),
slot_name: 'test',
hydratedSyncRules() {
return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(1) });
return this.sync_rules.config.hydrate({ hydrationState: versionedHydrationState(1) });
}
};
},
Expand Down
4 changes: 2 additions & 2 deletions packages/service-core/src/api/RouteAPI.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
import * as types from '@powersync/service-types';
import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';

Expand Down Expand Up @@ -41,7 +41,7 @@ export interface RouteAPI {
* tables to ensure syncing should function according to the input
* pattern. Debug errors and warnings are reported per table.
*/
getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SqlSyncRules): Promise<PatternResult[]>;
getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SyncConfig): Promise<PatternResult[]>;

/**
* @returns The replication lag: that is the amount of data which has not been
Expand Down
9 changes: 5 additions & 4 deletions packages/service-core/src/api/diagnostics.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { logger } from '@powersync/lib-services-framework';
import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules } from '@powersync/service-sync-rules';
import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules, SyncConfigWithErrors } from '@powersync/service-sync-rules';
import { ReplicationError, SyncRulesStatus, TableInfo } from '@powersync/service-types';

import * as storage from '../storage/storage-index.js';
Expand Down Expand Up @@ -41,11 +41,11 @@ export async function getSyncRulesStatus(
const check_connection = options.check_connection ?? false;
const now = new Date().toISOString();

let rules: SqlSyncRules;
let parsed: SyncConfigWithErrors;
let persisted: storage.PersistedSyncRules;
try {
persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
rules = persisted.sync_rules;
parsed = persisted.sync_rules;
} catch (e) {
return {
content: include_content ? sync_rules.sync_rules_content : undefined,
Expand All @@ -54,6 +54,7 @@ export async function getSyncRulesStatus(
};
}

const { config: rules, errors: syncRuleErrors } = parsed;
const sourceConfig = await apiHandler.getSourceConfig();
// This method can run under some situations if no connection is configured yet.
// It will return a default tag in such a case. This default tag is not module specific.
Expand Down Expand Up @@ -131,7 +132,7 @@ export async function getSyncRulesStatus(
});
}
errors.push(
...rules.errors.map((e) => {
...syncRuleErrors.map((e) => {
return {
level: e.type,
message: e.message,
Expand Down
4 changes: 2 additions & 2 deletions packages/service-core/src/routes/endpoints/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ export const reprocess = routeDefinition({
}

const new_rules = await activeBucketStorage.updateSyncRules({
content: active.sync_rules.content,
content: active.sync_rules.config.content,
// These sync rules already passed validation. But if the rules are not valid anymore due
// to a service change, we do want to report the error here.
validate: true
Expand Down Expand Up @@ -179,7 +179,7 @@ export const validate = routeDefinition({
schema
}),
hydratedSyncRules() {
return this.sync_rules.hydrate();
return this.sync_rules.config.hydrate();
}
};
},
Expand Down
10 changes: 5 additions & 5 deletions packages/service-core/src/routes/endpoints/sync-rules.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ export const reprocessSyncRules = routeDefinition({
}

const new_rules = await activeBucketStorage.updateSyncRules({
content: sync_rules.sync_rules.content,
content: sync_rules.sync_rules.config.content,
// These sync rules already passed validation. But if the rules are not valid anymore due
// to a service change, we do want to report the error here.
validate: true
Expand Down Expand Up @@ -197,14 +197,14 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
// No schema-based validation at this point
schema: undefined
});
const source_table_patterns = rules.getSourceTables();
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
const source_table_patterns = rules.config.getSourceTables();
const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules.config);

return {
valid: true,
bucket_definitions: rules.debugRepresentation(),
bucket_definitions: rules.config.debugRepresentation(),
source_tables: resolved_tables,
data_tables: rules.debugGetOutputTables()
data_tables: rules.config.debugGetOutputTables()
};
} catch (e) {
if (e instanceof SyncRulesErrors) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules';
import { HydratedSyncRules, SyncConfig, SyncConfigWithErrors } from '@powersync/service-sync-rules';
import { ReplicationLock } from './ReplicationLock.js';

export interface ParseSyncRulesOptions {
Expand Down Expand Up @@ -28,7 +28,7 @@ export interface PersistedSyncRulesContent {

export interface PersistedSyncRules {
readonly id: number;
readonly sync_rules: SqlSyncRules;
readonly sync_rules: SyncConfigWithErrors;
readonly slot_name: string;

hydratedSyncRules(): HydratedSyncRules;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ bucket_definitions:
data: []
`,
{ defaultSchema: 'public' }
).hydrate({ hydrationState: versionedHydrationState(1) });
).config.hydrate({ hydrationState: versionedHydrationState(1) });

// global[1] and global[2]
const SYNC_RULES_GLOBAL_TWO = SqlSyncRules.fromYaml(
Expand All @@ -39,7 +39,7 @@ bucket_definitions:
data: []
`,
{ defaultSchema: 'public' }
).hydrate({ hydrationState: versionedHydrationState(2) });
).config.hydrate({ hydrationState: versionedHydrationState(2) });

// by_project[n]
const SYNC_RULES_DYNAMIC = SqlSyncRules.fromYaml(
Expand All @@ -50,7 +50,7 @@ bucket_definitions:
data: []
`,
{ defaultSchema: 'public' }
).hydrate({ hydrationState: versionedHydrationState(3) });
).config.hydrate({ hydrationState: versionedHydrationState(3) });

const syncContext = new SyncContext({
maxBuckets: 100,
Expand Down Expand Up @@ -615,7 +615,7 @@ config:

const rules = SqlSyncRules.fromYaml(source, {
defaultSchema: 'public'
}).hydrate({ hydrationState: versionedHydrationState(1) });
}).config.hydrate({ hydrationState: versionedHydrationState(1) });

return new BucketChecksumState({
syncContext,
Expand Down
9 changes: 3 additions & 6 deletions packages/sync-rules/src/HydratedSyncRules.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import { Scope } from 'ajv/dist/compile/codegen/scope.js';
import { BucketDataSource, CreateSourceParams, HydratedBucketSource } from './BucketSource.js';
import { BucketDataScope, ParameterLookupScope } from './HydrationState.js';
import {
ParameterIndexLookupCreator,
BucketParameterQuerier,
buildBucketName,
CompatibilityContext,
EvaluatedParameters,
EvaluatedRow,
Expand All @@ -23,7 +20,7 @@ import {
SqlEventDescriptor,
SqliteInputValue,
SqliteValue,
SqlSyncRules
SyncConfig
} from './index.js';
import { SourceTableInterface } from './SourceTableInterface.js';
import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SqliteRow } from './types.js';
Expand All @@ -37,13 +34,13 @@ export class HydratedSyncRules {
eventDescriptors: SqlEventDescriptor[] = [];
compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY;

readonly definition: SqlSyncRules;
readonly definition: SyncConfig;

private readonly innerEvaluateRow: ScopedEvaluateRow;
private readonly innerEvaluateParameterRow: ScopedEvaluateParameterRow;

constructor(params: {
definition: SqlSyncRules;
definition: SyncConfig;
createParams: CreateSourceParams;
bucketDataSources: BucketDataSource[];
bucketParameterIndexLookupCreators: ParameterIndexLookupCreator[];
Expand Down
Loading