From 4aa3d98d28683bbd2029caee8fbd0367144ffd6f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 10:16:36 +0200 Subject: [PATCH 01/13] Initial EBNF grammar definitions, with validation. --- .../grammar/1-bucket-definitions.ebnf | 98 ++++++ .../grammar/2-sync-streams-alpha.ebnf | 93 +++++ .../grammar/3-sync-streams-compiler.ebnf | 115 +++++++ packages/sync-rules/package.json | 1 + .../fixtures/bucket_definitions.yaml | 36 ++ .../grammar_parity/fixtures/new_compiler.yaml | 33 ++ .../fixtures/sync_streams_alpha.yaml | 27 ++ .../src/grammar_parity/generated_grammar.ts | 110 ++++++ .../test/src/grammar_parity/parity.test.ts | 324 ++++++++++++++++++ pnpm-lock.yaml | 31 +- 10 files changed, 865 insertions(+), 3 deletions(-) create mode 100644 packages/sync-rules/grammar/1-bucket-definitions.ebnf create mode 100644 packages/sync-rules/grammar/2-sync-streams-alpha.ebnf create mode 100644 packages/sync-rules/grammar/3-sync-streams-compiler.ebnf create mode 100644 packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml create mode 100644 packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml create mode 100644 packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml create mode 100644 packages/sync-rules/test/src/grammar_parity/generated_grammar.ts create mode 100644 packages/sync-rules/test/src/grammar_parity/parity.test.ts diff --git a/packages/sync-rules/grammar/1-bucket-definitions.ebnf b/packages/sync-rules/grammar/1-bucket-definitions.ebnf new file mode 100644 index 000000000..19081c9a8 --- /dev/null +++ b/packages/sync-rules/grammar/1-bucket-definitions.ebnf @@ -0,0 +1,98 @@ +/* + PowerSync sync-rules SQL grammar: bucket_definitions mode (legacy parser path). + + SQL appears in YAML at: + - bucket_definitions..parameters + - string or array of strings + - bucket_definitions..data[] + + Source-of-truth files: + - src/SqlBucketDescriptor.ts + - src/SqlParameterQuery.ts + - src/StaticSqlParameterQuery.ts + - src/TableValuedFunctionSqlParameterQuery.ts + - src/SqlDataQuery.ts + - src/sql_filters.ts + - src/sql_support.ts +*/ + +BucketDefinitionSql ::= ParameterQuery | DataQuery + +/* Parameter-query SQL forms used under bucket_definitions..parameters */ +ParameterQuery ::= TableValuedParameterQuery | TableParameterQuery | StaticParameterQuery + +StaticParameterQuery ::= "SELECT" SelectList ("WHERE" ParameterExpr)? + +TableParameterQuery ::= "SELECT" SelectList "FROM" TableRef ("WHERE" MatchExpr)? + +/* Only json_each(...) table-valued FROM is supported in this mode. */ +TableValuedParameterQuery ::= "SELECT" SelectList "FROM" JsonEachCall Alias? ("WHERE" ParameterExpr)? + +JsonEachCall ::= "JSON_EACH" "(" ParameterExpr ")" + +/* Data-query SQL forms used under bucket_definitions..data[] */ +DataQuery ::= "SELECT" DataSelectList "FROM" TableRef ("WHERE" DataMatchExpr)? + +SelectList ::= SelectItem ("," SelectItem)* + +DataSelectList ::= DataSelectItem ("," DataSelectItem)* + +SelectItem ::= ScalarExpr Alias? + +DataSelectItem ::= "*" | ScalarExpr Alias? + +Alias ::= "AS" Identifier + +TableRef ::= Identifier ("." Identifier)? + +ParameterExpr ::= ScalarExpr + +DataMatchExpr ::= MatchExpr + +MatchExpr ::= OrExpr + +/* WHERE boolean expression subset */ +OrExpr ::= AndExpr ("OR" AndExpr)* + +AndExpr ::= UnaryExpr ("AND" UnaryExpr)* + +UnaryExpr ::= "NOT"? Predicate + +Predicate ::= ScalarExpr PredicateTail | ScalarExpr + +PredicateTail ::= "=" ScalarExpr | "IN" ScalarExpr | "&&" ScalarExpr | "IS" "NOT"? "NULL" + +/* Scalar expression subset (sql_filters/sql_functions) */ +ScalarExpr ::= ValueTerm (BinaryOp ValueTerm)* + +ValueTerm ::= PrimaryTerm MemberSuffix* + +PrimaryTerm ::= Literal | CastExpr | FunctionCall | Reference | "(" ScalarExpr ")" + +MemberSuffix ::= ("->" | "->>") (StringLiteral | IntegerLiteral) + +CastExpr ::= "CAST" "(" ScalarExpr "AS" CastType ")" + +FunctionCall ::= (Identifier ".")? Identifier "(" ArgList? ")" + +ArgList ::= ScalarExpr ("," ScalarExpr)* + +Reference ::= Identifier ("." Identifier)? + +BinaryOp ::= "=" | "!=" | "<" | ">" | "<=" | ">=" | "+" | "-" | "*" | "/" | "%" | "||" | "|" | "&" | "<<" | ">>" + +CastType ::= "TEXT" | "INTEGER" | "REAL" | "NUMERIC" | "BLOB" + +/* Identifiers are matched after SQL normalization to uppercase in tests. */ +Identifier ::= [A-Z_][A-Z_0-9]* + +Literal ::= StringLiteral | NumericLiteral | "TRUE" | "FALSE" | "NULL" + +StringLiteral ::= '"' ([#x20-#x21] | [#x23-#x5B] | [#x5D-#x7E])* '"' | "'" ([#x20-#x26] | [#x28-#x7E])* "'" + +IntegerLiteral ::= [0-9]+ + +NumericLiteral ::= [0-9]+ ("." [0-9]+)? + +/* Explicit whitespace rule used by the runtime parser adapter. */ +WS ::= [#x20#x09#x0A#x0D]+ diff --git a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf new file mode 100644 index 000000000..4e84ed478 --- /dev/null +++ b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf @@ -0,0 +1,93 @@ +/* + PowerSync sync-rules SQL grammar: sync streams alpha (syncStreamFromSql path, not new compiler). + + SQL appears in YAML at: + - streams..query + + Notes: + - This is the sync streams alpha SQL parser path. + - YAML keys `with` and `queries` are not supported in this mode. + + Source-of-truth files: + - src/streams/from_sql.ts + - src/streams/functions.ts + - src/sql_filters.ts + - src/sql_support.ts +*/ + +SyncStreamsAlphaSql ::= SyncStreamsAlphaQuery + +/* SQL form used under streams..query in sync streams alpha mode */ +SyncStreamsAlphaQuery ::= "SELECT" StreamSelectList "FROM" TableRef ("WHERE" StreamWhereExpr)? + +StreamSelectList ::= StreamSelectItem ("," StreamSelectItem)* + +StreamSelectItem ::= "*" | ScalarExpr Alias? + +StreamWhereExpr ::= OrExpr + +/* WHERE boolean expression subset */ +OrExpr ::= AndExpr ("OR" AndExpr)* + +AndExpr ::= UnaryExpr ("AND" UnaryExpr)* + +UnaryExpr ::= "NOT"? StreamPredicate + +StreamPredicate ::= ScalarExpr StreamPredicateTail | ScalarExpr + +/* IN/NOT IN/&& may target a scalar expression or a restricted subquery */ +StreamPredicateTail ::= "=" ScalarExpr | "IN" InSource | "NOT" "IN" InSource | "&&" InSource | "IS" "NOT"? "NULL" + +InSource ::= ScalarExpr | StreamSubquery + +StreamSubquery ::= "(" "SELECT" SubqueryResultExpr "FROM" TableRef ("WHERE" SubqueryWhereExpr)? ")" + +SubqueryResultExpr ::= ScalarExpr + +/* Sync streams alpha subquery filters are AND-only (no OR) */ +SubqueryWhereExpr ::= SubqueryAndExpr + +SubqueryAndExpr ::= SubqueryPredicate ("AND" SubqueryPredicate)* + +SubqueryPredicate ::= ScalarExpr SubqueryPredicateTail | ScalarExpr + +SubqueryPredicateTail ::= "=" ScalarExpr | "IN" ScalarExpr | "&&" ScalarExpr + +/* Scalar expression subset (sql_filters/sql_functions) */ +ScalarExpr ::= ValueTerm (BinaryOp ValueTerm)* + +ValueTerm ::= PrimaryTerm MemberSuffix* + +PrimaryTerm ::= Literal | CastExpr | FunctionCall | Reference | "(" ScalarExpr ")" + +MemberSuffix ::= ("->" | "->>") (StringLiteral | IntegerLiteral) + +CastExpr ::= "CAST" "(" ScalarExpr "AS" CastType ")" + +FunctionCall ::= (Identifier ".")? Identifier "(" ArgList? ")" + +ArgList ::= ScalarExpr ("," ScalarExpr)* + +Reference ::= Identifier ("." Identifier)? + +Alias ::= "AS" Identifier + +TableRef ::= Identifier ("." Identifier)? + +BinaryOp ::= "=" | "!=" | "<" | ">" | "<=" | ">=" | "+" | "-" | "*" | "/" | "%" | "||" | "|" | "&" | "<<" | ">>" + +CastType ::= "TEXT" | "INTEGER" | "REAL" | "NUMERIC" | "BLOB" + +/* Identifiers are matched after SQL normalization to uppercase in tests. */ +Identifier ::= [A-Z_][A-Z_0-9]* + +Literal ::= StringLiteral | NumericLiteral | "TRUE" | "FALSE" | "NULL" + +StringLiteral ::= '"' ([#x20-#x21] | [#x23-#x5B] | [#x5D-#x7E])* '"' | "'" ([#x20-#x26] | [#x28-#x7E])* "'" + +IntegerLiteral ::= [0-9]+ + +NumericLiteral ::= [0-9]+ ("." [0-9]+)? + +/* Explicit whitespace rule used by the runtime parser adapter. */ +WS ::= [#x20#x09#x0A#x0D]+ diff --git a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf new file mode 100644 index 000000000..4c645568f --- /dev/null +++ b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf @@ -0,0 +1,115 @@ +/* + PowerSync sync-rules SQL grammar: new sync streams compiler. + + SQL appears in YAML at: + - streams..query + - streams..queries[] + - streams..with. + + Notes: + - This grammar is used when config.sync_config_compiler = true (and edition >= 2). + - In this mode, bucket_definitions are rejected. + + Source-of-truth files: + - src/compiler/compiler.ts + - src/compiler/parser.ts + - src/compiler/sqlite.ts +*/ + +SyncStreamsCompilerSql ::= CompilerStreamQuery | CompilerSubquery | CompilerCteSubquery + +/* Top-level stream query form: streams..query / streams..queries[] */ +CompilerStreamQuery ::= "SELECT" ResultColumnList "FROM" FromSource FromContinuation* ("WHERE" WhereExpr)? + +ResultColumnList ::= ResultColumn ("," ResultColumn)* + +/* Supports wildcard, table wildcard, or scalar expression columns */ +ResultColumn ::= "*" | Reference "." "*" | ScalarExpr Alias? + +FromContinuation ::= "," FromSource | JoinClause + +FromSource ::= TableSource | TableValuedSource | SubquerySource + +/* Table and table-valued sources may be unaliased or AS-aliased */ +TableSource ::= TableRef Alias | TableRef + +TableValuedSource ::= TableValuedCall Alias | TableValuedCall + +/* Subqueries require an alias; optional column-name list is supported */ +SubquerySource ::= "(" CompilerSubquery ")" Alias ("(" ColumnNameList ")")? + +JoinClause ::= "INNER"? "JOIN" FromSource ("ON" WhereExpr)? + +TableValuedCall ::= Identifier "(" ArgList? ")" + +WhereExpr ::= OrExpr + +/* WHERE boolean expression subset */ +OrExpr ::= AndExpr ("OR" AndExpr)* + +AndExpr ::= UnaryExpr ("AND" UnaryExpr)* + +UnaryExpr ::= "NOT"? Predicate + +Predicate ::= ScalarExpr PredicateTail | ScalarExpr + +/* Includes IN/NOT IN, overlap, null checks, and BETWEEN */ +PredicateTail ::= "=" ScalarExpr | "IN" InSource | "NOT" "IN" InSource | "&&" InSource | "IS" "NOT"? "NULL" | "NOT"? "BETWEEN" ScalarExpr "AND" ScalarExpr + +InSource ::= "(" CompilerSubquery ")" | CteShorthandRef | ScalarExpr + +/* CTE shorthand supports expressions like: x IN cte_name */ +CteShorthandRef ::= Identifier + +CompilerSubquery ::= "SELECT" ResultColumnList "FROM" FromSource FromContinuation* ("WHERE" WhereExpr)? + +/* CTE bodies (streams..with.) disallow wildcard columns by using ScalarExpr columns only */ +CompilerCteSubquery ::= "SELECT" CteResultColumnList "FROM" FromSource FromContinuation* ("WHERE" WhereExpr)? + +CteResultColumnList ::= CteResultColumn ("," CteResultColumn)* + +CteResultColumn ::= ScalarExpr Alias? + +/* Scalar expression subset lowered by compiler/sqlite.ts */ +ScalarExpr ::= ValueTerm (BinaryOp ValueTerm)* + +ValueTerm ::= PrimaryTerm MemberSuffix* + +PrimaryTerm ::= Literal | CaseExpr | CastExpr | FunctionCall | Reference | "(" ScalarExpr ")" + +MemberSuffix ::= ("->" | "->>") (StringLiteral | IntegerLiteral) + +CastExpr ::= "CAST" "(" ScalarExpr "AS" CastType ")" + +/* CASE expression subset supported by compiler parser */ +CaseExpr ::= "CASE" ScalarExpr? "WHEN" ScalarExpr "THEN" ScalarExpr ("WHEN" ScalarExpr "THEN" ScalarExpr)* ("ELSE" ScalarExpr)? "END" + +FunctionCall ::= (Identifier ".")? Identifier "(" ArgList? ")" + +ArgList ::= ScalarExpr ("," ScalarExpr)* + +Reference ::= Identifier ("." Identifier)? + +Alias ::= "AS" Identifier + +TableRef ::= Identifier ("." Identifier)? + +ColumnNameList ::= Identifier ("," Identifier)* + +BinaryOp ::= "=" | "!=" | "<" | ">" | "<=" | ">=" | "+" | "-" | "*" | "/" | "%" | "||" | "|" | "&" | "<<" | ">>" + +CastType ::= "TEXT" | "INTEGER" | "REAL" | "NUMERIC" | "BLOB" + +/* Identifiers are matched after SQL normalization to uppercase in tests. */ +Identifier ::= [A-Z_][A-Z_0-9]* + +Literal ::= StringLiteral | NumericLiteral | "TRUE" | "FALSE" | "NULL" + +StringLiteral ::= '"' ([#x20-#x21] | [#x23-#x5B] | [#x5D-#x7E])* '"' | "'" ([#x20-#x26] | [#x28-#x7E])* "'" + +IntegerLiteral ::= [0-9]+ + +NumericLiteral ::= [0-9]+ ("." [0-9]+)? + +/* Explicit whitespace rule used by the runtime parser adapter. */ +WS ::= [#x20#x09#x0A#x0D]+ diff --git a/packages/sync-rules/package.json b/packages/sync-rules/package.json index bdf5f3131..29ac13abf 100644 --- a/packages/sync-rules/package.json +++ b/packages/sync-rules/package.json @@ -31,6 +31,7 @@ }, "devDependencies": { "@types/node": "^22.16.2", + "ebnf": "^1.9.1", "vitest": "catalog:" } } diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml new file mode 100644 index 000000000..18badf6de --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml @@ -0,0 +1,36 @@ +parameters: + accepted: + - sql: SELECT token_parameters.user_id AS user_id + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id + - sql: SELECT json_each.value AS tag FROM json_each(request.parameters() -> 'tags') + + rejected_syntax: + # table aliases are not supported in parameter queries + - sql: SELECT u.org_id AS org_id FROM users u WHERE u.id = token_parameters.user_id + err: alias + + # only json_each(...) table-valued functions are supported + - sql: SELECT x FROM generate_series(1, 3) AS x + err: not defined + + - sql: SELECT token_parameters.user_id AS user_id LIMIT 1 + err: LIMIT + +data: + accepted: + - sql: SELECT id FROM assets + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id + params: [user_id] + + rejected_syntax: + - sql: SELECT assets.id FROM assets JOIN users ON users.id = assets.owner_id + err: single table + + rejected_semantic: + - sql: SELECT upper(name) FROM assets + err: alias + + # missing owner_id = bucket.user_id condition + - sql: SELECT id FROM assets + params: [user_id] + err: cover all bucket parameters diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml new file mode 100644 index 000000000..e2e276c31 --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml @@ -0,0 +1,33 @@ +query: + accepted: + - sql: SELECT * FROM users + - sql: SELECT u.* FROM users AS u JOIN orgs AS o ON u.org_id = o.id WHERE o.owner_id = auth.user_id() + + rejected_syntax: + - sql: SELECT u.* FROM users u FULL JOIN orgs o ON u.org_id = o.id + err: JOIN + + - sql: SELECT u.* FROM users u INNER JOIN orgs USING (org_id) + err: USING + + - sql: SELECT * FROM users ORDER BY id + err: ORDER BY + + - sql: SELECT * FROM users WHERE id = $1 + err: parameters are not allowed + + - sql: DELETE FROM users + err: SELECT + + rejected_semantic: + - sql: SELECT u.*, auth.parameter('x') FROM users AS u + err: connection parameter + +with: + accepted: + - sql: SELECT id FROM orgs WHERE owner_id = auth.user_id() + + rejected_syntax: + # CTE output columns cannot include wildcard + - sql: SELECT * FROM orgs + err: '*' diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml new file mode 100644 index 000000000..e80764e18 --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml @@ -0,0 +1,27 @@ +query: + accepted: + - sql: SELECT * FROM comments + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) + + rejected_syntax: + - sql: SELECT id FROM comments GROUP BY id + err: GROUP BY + + - sql: INSERT INTO comments (id) VALUES ('c1') + err: SELECT + + - sql: SELECT * FROM comments; SELECT * FROM issues + err: single + + - sql: SELECT * FROM json_each(auth.parameter('x')) + err: single table + + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() OR name = 'test') + err: OR + + rejected_semantic: + - sql: SELECT * FROM comments WHERE 'static' IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) + err: left operand + + - sql: SELECT * FROM issues WHERE owner_id = request.user_id() + err: not defined diff --git a/packages/sync-rules/test/src/grammar_parity/generated_grammar.ts b/packages/sync-rules/test/src/grammar_parity/generated_grammar.ts new file mode 100644 index 000000000..cdbbc3f3f --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/generated_grammar.ts @@ -0,0 +1,110 @@ +import { readFileSync } from 'node:fs'; +import { Grammars } from 'ebnf'; + +interface AstNode { + errors?: unknown[]; +} + +interface GrammarParser { + getAST(input: string, target?: string): AstNode | null; +} + +const parserCache = new Map(); + +export function grammarAcceptsSql(grammarFilePath: string, startRule: string, sql: string): boolean { + try { + const parser = getOrCreateParser(grammarFilePath, startRule); + const normalizedInput = normalizeSqlForGrammar(sql); + const ast = parser.getAST(normalizedInput, 'ENTRY'); + + return ast != null && (ast.errors?.length ?? 0) === 0; + } catch { + return false; + } +} + +function getOrCreateParser(grammarFilePath: string, startRule: string): GrammarParser { + const cacheKey = `${grammarFilePath}::${startRule}`; + const cached = parserCache.get(cacheKey); + if (cached != null) { + return cached; + } + + const source = `${readFileSync(grammarFilePath, 'utf8')}\nENTRY ::= ${startRule} EOF\n`; + const parser = new Grammars.W3C.Parser(source); + for (const rule of (parser as any).grammarRules ?? []) { + if ( + rule?.name !== 'WS' && + rule?.name !== 'Identifier' && + rule?.name !== 'IntegerLiteral' && + rule?.name !== 'NumericLiteral' && + rule?.name !== 'StringLiteral' && + !(typeof rule?.name === 'string' && rule.name.startsWith('%')) + ) { + rule.implicitWs = true; + } + } + + parserCache.set(cacheKey, parser); + return parser; +} + +function normalizeSqlForGrammar(sql: string): string { + let result = ''; + let inSingle = false; + let inDouble = false; + let lastWasSpace = false; + + for (let i = 0; i < sql.length; i++) { + const char = sql[i]; + + if (inSingle) { + result += char; + + if (char === "'" && sql[i + 1] === "'") { + result += sql[i + 1]; + i++; + } else if (char === "'") { + inSingle = false; + } + continue; + } + + if (inDouble) { + result += char; + + if (char === '"' && sql[i + 1] === '"') { + result += sql[i + 1]; + i++; + } else if (char === '"') { + inDouble = false; + } + continue; + } + + if (char === "'") { + inSingle = true; + result += char; + continue; + } + + if (char === '"') { + inDouble = true; + result += char; + continue; + } + + if (/\s/.test(char)) { + if (!lastWasSpace) { + result += ' '; + lastWasSpace = true; + } + continue; + } + + result += char.toUpperCase(); + lastWasSpace = false; + } + + return result.trim(); +} diff --git a/packages/sync-rules/test/src/grammar_parity/parity.test.ts b/packages/sync-rules/test/src/grammar_parity/parity.test.ts new file mode 100644 index 000000000..84ab5e199 --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/parity.test.ts @@ -0,0 +1,324 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { parse as parseYaml } from 'yaml'; +import { describe, expect, test } from 'vitest'; +import { + CompatibilityContext, + CompatibilityEdition, + SqlDataQuery, + SqlParameterQuery, + syncStreamFromSql, + SyncStreamsCompiler +} from '../../../src/index.js'; +import { EMPTY_DATA_SOURCE, PARSE_OPTIONS } from '../util.js'; +import { grammarAcceptsSql } from './generated_grammar.js'; + +type FixtureMode = 'bucket_definitions' | 'sync_streams_alpha' | 'new_compiler'; +type BucketSlot = 'parameters' | 'data'; +type StreamSlot = 'query'; +type CompilerSlot = 'query' | 'with'; +type FixtureSlot = BucketSlot | StreamSlot | CompilerSlot; + +type FixtureKind = 'accepted' | 'rejected_syntax' | 'rejected_semantic'; + +interface FixtureCase { + label: string; + mode: FixtureMode; + slot: FixtureSlot; + kind: FixtureKind; + sql: string; + parserOk: boolean; + grammarOk: boolean; + err?: string; + params?: string[]; +} + +interface FixtureEntry { + sql: string; + err?: string; + params?: string[]; +} + +interface FixtureGroup { + accepted?: FixtureEntry[]; + rejected_syntax?: FixtureEntry[]; + rejected_semantic?: FixtureEntry[]; +} + +type FixtureFile = Partial>; + +interface Outcome { + accept: boolean; + messages: string[]; +} + +interface ListenerError { + message: string; + isWarning: boolean; +} + +const GRAMMAR_FILE_BY_MODE: Record = { + bucket_definitions: fileURLToPath(new URL('../../../grammar/1-bucket-definitions.ebnf', import.meta.url)), + sync_streams_alpha: fileURLToPath(new URL('../../../grammar/2-sync-streams-alpha.ebnf', import.meta.url)), + new_compiler: fileURLToPath(new URL('../../../grammar/3-sync-streams-compiler.ebnf', import.meta.url)) +}; + +const fixtures: FixtureCase[] = [ + ...loadFixtureFile('fixtures/bucket_definitions.yaml', 'bucket_definitions'), + ...loadFixtureFile('fixtures/sync_streams_alpha.yaml', 'sync_streams_alpha'), + ...loadFixtureFile('fixtures/new_compiler.yaml', 'new_compiler') +]; + +describe('grammar parity fixtures', () => { + test.each(fixtures)('parser contract: $mode/$slot/$kind/$label', (fixture) => { + const outcome = runParser(fixture); + assertParserExpectation(fixture, outcome); + }); + + test.each(fixtures)('grammar contract: $mode/$slot/$kind/$label', (fixture) => { + const outcome = runGrammarChecker(fixture); + assertGrammarExpectation(fixture, outcome); + }); + + test.each(fixtures)('parser/grammar matrix: $mode/$slot/$kind/$label', (fixture) => { + const parserOutcome = runParser(fixture); + const grammarOutcome = runGrammarChecker(fixture); + + expect( + { + parser: parserOutcome.accept, + grammar: grammarOutcome.accept + }, + `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` + ).toEqual({ + parser: fixture.parserOk, + grammar: fixture.grammarOk + }); + }); +}); + +function loadFixtureFile(relativePath: string, mode: FixtureMode): FixtureCase[] { + const filePath = fileURLToPath(new URL(relativePath, import.meta.url)); + const parsed = parseYaml(readFileSync(filePath, 'utf8')) as FixtureFile; + const output: FixtureCase[] = []; + + for (const [slot, group] of Object.entries(parsed) as Array<[FixtureSlot, FixtureGroup | undefined]>) { + if (group == null) { + continue; + } + + pushGroup(output, mode, slot, 'accepted', group.accepted ?? []); + pushGroup(output, mode, slot, 'rejected_syntax', group.rejected_syntax ?? []); + pushGroup(output, mode, slot, 'rejected_semantic', group.rejected_semantic ?? []); + } + + return output; +} + +function pushGroup( + out: FixtureCase[], + mode: FixtureMode, + slot: FixtureSlot, + kind: FixtureKind, + entries: FixtureEntry[] +): void { + entries.forEach((entry, index) => { + const expected = expectedOutcomes(kind); + + out.push({ + mode, + slot, + kind, + sql: entry.sql, + err: entry.err, + params: entry.params, + parserOk: expected.parserOk, + grammarOk: expected.grammarOk, + label: fixtureLabel(entry.sql, index) + }); + }); +} + +function expectedOutcomes(kind: FixtureKind): { parserOk: boolean; grammarOk: boolean } { + switch (kind) { + case 'accepted': + return { parserOk: true, grammarOk: true }; + case 'rejected_syntax': + return { parserOk: false, grammarOk: false }; + case 'rejected_semantic': + return { parserOk: false, grammarOk: true }; + } +} + +function assertParserExpectation(fixture: FixtureCase, outcome: Outcome): void { + expect( + { + mode: fixture.mode, + slot: fixture.slot, + kind: fixture.kind, + sql: fixture.sql, + expected: fixture.parserOk, + actual: outcome.accept, + messages: outcome.messages + }, + `Parser expectation mismatch for ${fixtureRef(fixture)}` + ).toMatchObject({ actual: fixture.parserOk }); + + if (!fixture.parserOk && fixture.err) { + const needle = fixture.err.toLowerCase(); + expect( + outcome.messages.some((message) => message.toLowerCase().includes(needle)), + `Expected a parser error containing '${fixture.err}' for ${fixtureRef(fixture)}` + ).toBe(true); + } +} + +function assertGrammarExpectation(fixture: FixtureCase, outcome: Outcome): void { + expect( + { + mode: fixture.mode, + slot: fixture.slot, + kind: fixture.kind, + sql: fixture.sql, + expected: fixture.grammarOk, + actual: outcome.accept, + messages: outcome.messages + }, + `Grammar expectation mismatch for ${fixtureRef(fixture)}` + ).toMatchObject({ actual: fixture.grammarOk }); +} + +function fixtureLabel(sql: string, index: number): string { + const oneLine = sql.replace(/\s+/g, ' ').trim(); + const clipped = oneLine.length > 60 ? `${oneLine.slice(0, 57)}...` : oneLine; + return `${index + 1}. ${clipped}`; +} + +function fixtureRef(fixture: FixtureCase): string { + return `${fixture.mode}/${fixture.slot}/${fixture.kind}/${fixture.label}`; +} + +function runParser(fixture: FixtureCase): Outcome { + switch (fixture.mode) { + case 'bucket_definitions': + return runBucketParser(fixture); + case 'sync_streams_alpha': + return runSyncStreamsAlphaParser(fixture); + case 'new_compiler': + return runNewCompilerParser(fixture); + } +} + +function runBucketParser(fixture: FixtureCase): Outcome { + const slot = fixture.slot as BucketSlot; + + try { + if (slot === 'parameters') { + const parsed = SqlParameterQuery.fromSql( + 'bucket', + fixture.sql, + { + defaultSchema: PARSE_OPTIONS.defaultSchema, + compatibility: PARSE_OPTIONS.compatibility, + accept_potentially_dangerous_queries: true + }, + 'parity-query', + EMPTY_DATA_SOURCE + ); + + const messages = hardParserMessages(parsed.errors); + return { accept: messages.length === 0, messages }; + } + + const parsed = SqlDataQuery.fromSql( + fixture.params ?? [], + fixture.sql, + { defaultSchema: PARSE_OPTIONS.defaultSchema }, + PARSE_OPTIONS.compatibility + ); + + const messages = hardParserMessages(parsed.errors); + return { accept: messages.length === 0, messages }; + } catch (e) { + return rejectFromException(e); + } +} + +function runSyncStreamsAlphaParser(fixture: FixtureCase): Outcome { + try { + const [_, errors] = syncStreamFromSql('stream', fixture.sql, { + defaultSchema: PARSE_OPTIONS.defaultSchema, + compatibility: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }), + auto_subscribe: false + }); + + const messages = hardParserMessages(errors); + return { accept: messages.length === 0, messages }; + } catch (e) { + return rejectFromException(e); + } +} + +function runNewCompilerParser(fixture: FixtureCase): Outcome { + const compiler = new SyncStreamsCompiler({ defaultSchema: PARSE_OPTIONS.defaultSchema }); + const errors: ListenerError[] = []; + + const listener = { + report(message: string, _location: unknown, options?: { isWarning: boolean }) { + errors.push({ message, isWarning: options?.isWarning === true }); + } + }; + + try { + if (fixture.slot === 'with') { + const parsed = compiler.commonTableExpression(fixture.sql, listener); + if (parsed == null && errors.length === 0) { + errors.push({ message: 'Could not parse CTE SQL.', isWarning: false }); + } + } else { + const stream = compiler.stream({ name: 'stream', isSubscribedByDefault: false, priority: 3 }); + stream.addQuery(fixture.sql, listener); + stream.finish(); + } + + const messages = errors.filter((e) => !e.isWarning).map((e) => e.message); + return { accept: messages.length === 0, messages }; + } catch (e) { + return rejectFromException(e); + } +} + +function runGrammarChecker(fixture: FixtureCase): Outcome { + const grammarPath = GRAMMAR_FILE_BY_MODE[fixture.mode]; + const startRule = grammarStartRule(fixture.mode, fixture.slot); + + const accept = grammarAcceptsSql(grammarPath, startRule, fixture.sql); + return { + accept, + messages: accept ? [] : [`Grammar ${startRule} did not accept SQL.`] + }; +} + +function grammarStartRule(mode: FixtureMode, slot: FixtureSlot): string { + if (mode === 'bucket_definitions') { + return slot === 'parameters' ? 'ParameterQuery' : 'DataQuery'; + } + + if (mode === 'sync_streams_alpha') { + return 'SyncStreamsAlphaQuery'; + } + + return slot === 'with' ? 'CompilerCteSubquery' : 'CompilerStreamQuery'; +} + +function hardParserMessages(errors: Array<{ message: string; type?: string }>): string[] { + return errors.filter((error) => error.type !== 'warning').map((error) => error.message); +} + +function rejectFromException(error: unknown): Outcome { + if (error instanceof Error) { + return { accept: false, messages: [error.message] }; + } + + return { accept: false, messages: [String(error)] }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c1acfeb0c..65a6ef937 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -733,6 +733,9 @@ importers: '@types/node': specifier: ^22.16.2 version: 22.16.2 + ebnf: + specifier: ^1.9.1 + version: 1.9.1 vitest: specifier: 'catalog:' version: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.4.5) @@ -3427,6 +3430,10 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ebnf@1.9.1: + resolution: {integrity: sha512-uW2UKSsuty9ANJ3YByIQE4ANkD8nqUPO7r6Fwcc1ADKPe9FRdcPpMl3VEput4JSvKBJ4J86npIC2MLP0pYkCuw==} + hasBin: true + ecdsa-sig-formatter@1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} @@ -8756,6 +8763,14 @@ snapshots: optionalDependencies: vite: 7.1.5(@types/node@24.10.9)(yaml@2.8.2) + '@vitest/mocker@4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.4.5))': + dependencies: + '@vitest/spy': 4.0.16 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.1.5(@types/node@22.16.2)(yaml@2.4.5) + '@vitest/mocker@4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.16 @@ -8764,6 +8779,14 @@ snapshots: optionalDependencies: vite: 7.1.5(@types/node@22.16.2)(yaml@2.8.2) + '@vitest/mocker@4.0.16(vite@7.1.5(@types/node@24.10.9)(yaml@2.8.2))': + dependencies: + '@vitest/spy': 4.0.16 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.1.5(@types/node@24.10.9)(yaml@2.8.2) + '@vitest/pretty-format@3.2.4': dependencies: tinyrainbow: 2.0.0 @@ -8810,7 +8833,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vitest: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.8.2) + vitest: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.4.5) '@vitest/utils@3.2.4': dependencies: @@ -9556,6 +9579,8 @@ snapshots: eastasianwidth@0.2.0: {} + ebnf@1.9.1: {} + ecdsa-sig-formatter@1.0.11: dependencies: safe-buffer: 5.2.1 @@ -12349,7 +12374,7 @@ snapshots: vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.4.5): dependencies: '@vitest/expect': 4.0.16 - '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.8.2)) + '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.4.5)) '@vitest/pretty-format': 4.0.16 '@vitest/runner': 4.0.16 '@vitest/snapshot': 4.0.16 @@ -12427,7 +12452,7 @@ snapshots: vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@24.10.9)(@vitest/ui@4.0.16)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.16 - '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.8.2)) + '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@24.10.9)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.16 '@vitest/runner': 4.0.16 '@vitest/snapshot': 4.0.16 From 26b366cfb4e8012a4714c9117654f5057a94051f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 11:51:29 +0200 Subject: [PATCH 02/13] Expand tests. --- .../bucket_definitions_parity.test.ts | 39 ++++ .../fixtures/bucket_definitions.yaml | 59 ++++++ .../grammar_parity/fixtures/new_compiler.yaml | 46 +++++ .../fixtures/sync_streams_alpha.yaml | 27 +++ .../new_compiler_parity.test.ts | 39 ++++ .../{parity.test.ts => parity_helpers.ts} | 192 +++++++----------- .../sync_streams_alpha_parity.test.ts | 39 ++++ 7 files changed, 328 insertions(+), 113 deletions(-) create mode 100644 packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts create mode 100644 packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts rename packages/sync-rules/test/src/grammar_parity/{parity.test.ts => parity_helpers.ts} (77%) create mode 100644 packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts diff --git a/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts new file mode 100644 index 000000000..c8cd104dc --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from 'vitest'; +import { + assertGrammarExpectation, + assertParserExpectation, + fixtureRef, + loadFixtureFile, + runGrammarChecker, + runParser +} from './parity_helpers.js'; + +const fixtures = loadFixtureFile('fixtures/bucket_definitions.yaml', 'bucket_definitions'); + +describe('grammar parity fixtures: bucket_definitions', () => { + test.each(fixtures)('parser contract: $slot/$kind/$label', (fixture) => { + const outcome = runParser(fixture); + assertParserExpectation(fixture, outcome); + }); + + test.each(fixtures)('grammar contract: $slot/$kind/$label', (fixture) => { + const outcome = runGrammarChecker(fixture); + assertGrammarExpectation(fixture, outcome); + }); + + test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { + const parserOutcome = runParser(fixture); + const grammarOutcome = runGrammarChecker(fixture); + + expect( + { + parser: parserOutcome.accept, + grammar: grammarOutcome.accept + }, + `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` + ).toEqual({ + parser: fixture.parserOk, + grammar: fixture.grammarOk + }); + }); +}); diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml index 18badf6de..5d099e510 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml @@ -3,16 +3,38 @@ parameters: - sql: SELECT token_parameters.user_id AS user_id - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id - sql: SELECT json_each.value AS tag FROM json_each(request.parameters() -> 'tags') + - sql: SELECT request.user_id() AS user_id + - sql: SELECT request.parameter('tenant_id') AS tenant_id + - sql: SELECT request.parameters() ->> 'org_id' AS org_id + - sql: SELECT request.jwt() ->> 'sub' AS sub + - sql: SELECT CAST(token_parameters.user_id AS text) AS user_id + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = request.user_id() + - sql: SELECT users.org_id AS org_id FROM test_schema.users WHERE users.id = token_parameters.user_id + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = request.parameter('user_id') + - sql: SELECT json_each.value AS tag FROM json_each(request.parameter('tags')) + - sql: SELECT json_each.value AS tag FROM json_each(request.parameters() -> 'tags') WHERE json_each.value IS NOT NULL + - sql: SELECT token_parameters.user_id || '' AS user_id + - sql: SELECT (token_parameters.user_id) AS user_id + - sql: SELECT request.parameters() -> 'profile' AS profile + - sql: SELECT request.parameter('features') ->> 'beta' AS beta + - sql: SELECT CAST(request.parameter('limit') AS integer) AS limit_value + - sql: SELECT token_parameters.user_id AS user_id WHERE request.user_id() IS NOT NULL + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id AND token_parameters.user_id IS NOT NULL + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id OR users.id = request.user_id() + - sql: SELECT CAST(request.parameters() ->> 'org_id' AS text) AS org_id rejected_syntax: # table aliases are not supported in parameter queries + # parser: Table aliases not supported in parameter queries - sql: SELECT u.org_id AS org_id FROM users u WHERE u.id = token_parameters.user_id err: alias # only json_each(...) table-valued functions are supported + # parser: Table-valued function generate_series is not defined. - sql: SELECT x FROM generate_series(1, 3) AS x err: not defined + # parser: LIMIT is not supported - sql: SELECT token_parameters.user_id AS user_id LIMIT 1 err: LIMIT @@ -21,16 +43,53 @@ data: - sql: SELECT id FROM assets - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id params: [user_id] + - sql: SELECT id, name FROM assets WHERE owner_id = bucket.user_id + params: [user_id] + - sql: SELECT assets.id AS id FROM assets WHERE assets.owner_id = bucket.user_id + params: [user_id] + - sql: SELECT id FROM assets WHERE bucket.user_id = owner_id + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND id IS NOT NULL + params: [user_id] + - sql: SELECT id FROM assets WHERE NOT (owner_id != bucket.user_id) + params: [user_id] + - sql: SELECT id, name AS asset_name FROM assets WHERE owner_id = bucket.user_id + params: [user_id] + - sql: SELECT id FROM test_schema.assets WHERE owner_id = bucket.user_id + params: [user_id] + - sql: SELECT id FROM assets WHERE (owner_id = bucket.user_id) + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id OR bucket.user_id = owner_id + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (name IS NOT NULL) + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND id = id + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (bucket.user_id IS NOT NULL) + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (owner_id = bucket.user_id) + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (owner_id IS NOT NULL) + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (name = name) + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (id = id) + params: [user_id] + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (owner_id = bucket.user_id OR owner_id = bucket.user_id) + params: [user_id] rejected_syntax: + # parser: Must SELECT from a single table - sql: SELECT assets.id FROM assets JOIN users ON users.id = assets.owner_id err: single table rejected_semantic: + # parser: alias is required - sql: SELECT upper(name) FROM assets err: alias # missing owner_id = bucket.user_id condition + # parser: Query must cover all bucket parameters. Expected: ["bucket.user_id"] Got: [] - sql: SELECT id FROM assets params: [user_id] err: cover all bucket parameters diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml index e2e276c31..171c41a36 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml @@ -2,32 +2,78 @@ query: accepted: - sql: SELECT * FROM users - sql: SELECT u.* FROM users AS u JOIN orgs AS o ON u.org_id = o.id WHERE o.owner_id = auth.user_id() + - sql: SELECT id FROM users + - sql: SELECT u.id FROM users AS u + - sql: SELECT u.id, u.org_id FROM users AS u + - sql: SELECT CAST(u.id AS text) AS id_text FROM users AS u + - sql: SELECT CASE WHEN u.id IS NULL THEN 'x' ELSE 'y' END AS state FROM users AS u + - sql: SELECT u.* FROM users AS u INNER JOIN orgs AS o ON u.org_id = o.id + - sql: SELECT u.* FROM users AS u JOIN orgs AS o ON u.org_id = o.id + - sql: SELECT u.* FROM users AS u JOIN orgs AS o ON u.org_id = o.id WHERE o.owner_id = auth.parameter('owner_id') + - sql: SELECT * FROM users WHERE id IS NOT NULL + - sql: SELECT * FROM users WHERE id = id + - sql: SELECT * FROM users WHERE id IN (SELECT id FROM users) + - sql: SELECT * FROM users WHERE id IN (SELECT id FROM users WHERE id IS NOT NULL) + - sql: SELECT * FROM users WHERE id NOT IN (SELECT id FROM users WHERE id IS NOT NULL) + - sql: SELECT * FROM users WHERE id BETWEEN 1 AND 10 + - sql: SELECT * FROM users WHERE id NOT BETWEEN 1 AND 10 + - sql: SELECT * FROM users WHERE auth.user_id() IS NOT NULL + - sql: SELECT * FROM users WHERE auth.parameter('tenant') IS NOT NULL + - sql: SELECT * FROM users WHERE subscription.parameter('plan') IS NOT NULL + - sql: SELECT * FROM users WHERE connection.parameter('id') IS NOT NULL + - sql: SELECT j.value AS value FROM json_each(auth.parameters()) AS j + - sql: SELECT u.id FROM users AS u WHERE (u.id = u.id) + - sql: SELECT u.id FROM users AS u WHERE (u.id = auth.parameter('id')) + - sql: SELECT u.id FROM users AS u WHERE (u.id = connection.parameter('id')) + - sql: SELECT u.id FROM users AS u WHERE (u.id = subscription.parameter('id')) + - sql: SELECT u.id, o.id FROM users AS u JOIN orgs AS o ON u.org_id = o.id + - sql: SELECT u.id FROM users AS u WHERE u.id IN (SELECT id FROM users AS x) + - sql: SELECT * FROM users WHERE id && (SELECT id FROM users WHERE id IS NOT NULL) rejected_syntax: + # parser: FULL JOIN is not supported - sql: SELECT u.* FROM users u FULL JOIN orgs o ON u.org_id = o.id err: JOIN + # parser: USING is not supported - sql: SELECT u.* FROM users u INNER JOIN orgs USING (org_id) err: USING + # parser: ORDER BY is not supported - sql: SELECT * FROM users ORDER BY id err: ORDER BY + # parser: SQL parameters are not allowed. Use parameter functions instead: https://docs.powersync.com/usage/sync-streams#accessing-parameters - sql: SELECT * FROM users WHERE id = $1 err: parameters are not allowed + # parser: Expected a SELECT statement - sql: DELETE FROM users err: SELECT rejected_semantic: + # parser: This attempts to sync a connection parameter. Only values from the source database can be synced. - sql: SELECT u.*, auth.parameter('x') FROM users AS u err: connection parameter with: accepted: - sql: SELECT id FROM orgs WHERE owner_id = auth.user_id() + - sql: SELECT id FROM orgs + - sql: SELECT id FROM orgs WHERE id IS NOT NULL + - sql: SELECT CAST(id AS text) AS id_text FROM orgs + - sql: SELECT CASE WHEN id IS NULL THEN 'x' ELSE 'y' END AS state FROM orgs + - sql: SELECT o.id FROM orgs AS o + - sql: SELECT o.id FROM orgs AS o WHERE o.owner_id = auth.user_id() + - sql: SELECT o.id FROM orgs AS o WHERE o.owner_id = auth.parameter('owner_id') + - sql: SELECT o.id FROM orgs AS o WHERE o.owner_id = connection.parameter('owner_id') + - sql: SELECT o.id FROM orgs AS o WHERE o.owner_id = subscription.parameter('owner_id') + - sql: SELECT o.id FROM orgs AS o WHERE o.id IN (SELECT id FROM orgs AS x) + - sql: SELECT o.id FROM orgs AS o WHERE o.id NOT IN (SELECT id FROM orgs AS x) + - sql: SELECT o.id FROM orgs AS o WHERE o.id BETWEEN 1 AND 10 rejected_syntax: # CTE output columns cannot include wildcard + # parser: * columns are not allowed in subqueries or common table expressions - sql: SELECT * FROM orgs err: '*' diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml index e80764e18..d88e0e6ec 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml @@ -2,26 +2,53 @@ query: accepted: - sql: SELECT * FROM comments - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) + - sql: SELECT id FROM comments + - sql: SELECT comments.id FROM comments + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND id IS NOT NULL) + - sql: SELECT * FROM comments WHERE issue_id NOT IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) + - sql: SELECT * FROM comments WHERE auth.user_id() IS NOT NULL + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues) + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.parameter('owner_id')) + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = connection.parameter('owner_id')) + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = subscription.parameter('owner_id')) + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND owner_id = auth.user_id()) + - sql: SELECT * FROM comments WHERE issue_id = issue_id + - sql: SELECT * FROM comments WHERE NOT (issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id())) + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND name = 'x') + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND id = id) + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM test_schema.issues WHERE owner_id = auth.user_id()) + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND auth.user_id() IS NOT NULL + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND connection.parameter('x') IS NULL + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND subscription.parameter('plan') IS NOT NULL + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND auth.parameter('tenant') IS NOT NULL + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND id = id rejected_syntax: + # parser: GROUP BY is not supported - sql: SELECT id FROM comments GROUP BY id err: GROUP BY + # parser: Only SELECT statements are supported - sql: INSERT INTO comments (id) VALUES ('c1') err: SELECT + # parser: Only a single SELECT statement is supported - sql: SELECT * FROM comments; SELECT * FROM issues err: single + # parser: Must SELECT from a single table - sql: SELECT * FROM json_each(auth.parameter('x')) err: single table + # parser: Stream subqueries can't use OR filters - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() OR name = 'test') err: OR rejected_semantic: + # parser: For IN subqueries, the left operand must either depend on the row to sync or stream parameters. - sql: SELECT * FROM comments WHERE 'static' IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) err: left operand + # parser: Function 'request.user_id' is not defined - sql: SELECT * FROM issues WHERE owner_id = request.user_id() err: not defined diff --git a/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts new file mode 100644 index 000000000..b5499e3ca --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from 'vitest'; +import { + assertGrammarExpectation, + assertParserExpectation, + fixtureRef, + loadFixtureFile, + runGrammarChecker, + runParser +} from './parity_helpers.js'; + +const fixtures = loadFixtureFile('fixtures/new_compiler.yaml', 'new_compiler'); + +describe('grammar parity fixtures: new_compiler', () => { + test.each(fixtures)('parser contract: $slot/$kind/$label', (fixture) => { + const outcome = runParser(fixture); + assertParserExpectation(fixture, outcome); + }); + + test.each(fixtures)('grammar contract: $slot/$kind/$label', (fixture) => { + const outcome = runGrammarChecker(fixture); + assertGrammarExpectation(fixture, outcome); + }); + + test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { + const parserOutcome = runParser(fixture); + const grammarOutcome = runGrammarChecker(fixture); + + expect( + { + parser: parserOutcome.accept, + grammar: grammarOutcome.accept + }, + `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` + ).toEqual({ + parser: fixture.parserOk, + grammar: fixture.grammarOk + }); + }); +}); diff --git a/packages/sync-rules/test/src/grammar_parity/parity.test.ts b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts similarity index 77% rename from packages/sync-rules/test/src/grammar_parity/parity.test.ts rename to packages/sync-rules/test/src/grammar_parity/parity_helpers.ts index 84ab5e199..bc248b979 100644 --- a/packages/sync-rules/test/src/grammar_parity/parity.test.ts +++ b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts @@ -1,7 +1,6 @@ import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { parse as parseYaml } from 'yaml'; -import { describe, expect, test } from 'vitest'; import { CompatibilityContext, CompatibilityEdition, @@ -12,16 +11,17 @@ import { } from '../../../src/index.js'; import { EMPTY_DATA_SOURCE, PARSE_OPTIONS } from '../util.js'; import { grammarAcceptsSql } from './generated_grammar.js'; +import { expect } from 'vitest'; -type FixtureMode = 'bucket_definitions' | 'sync_streams_alpha' | 'new_compiler'; -type BucketSlot = 'parameters' | 'data'; -type StreamSlot = 'query'; -type CompilerSlot = 'query' | 'with'; -type FixtureSlot = BucketSlot | StreamSlot | CompilerSlot; +export type FixtureMode = 'bucket_definitions' | 'sync_streams_alpha' | 'new_compiler'; +export type BucketSlot = 'parameters' | 'data'; +export type StreamSlot = 'query'; +export type CompilerSlot = 'query' | 'with'; +export type FixtureSlot = BucketSlot | StreamSlot | CompilerSlot; -type FixtureKind = 'accepted' | 'rejected_syntax' | 'rejected_semantic'; +export type FixtureKind = 'accepted' | 'rejected_syntax' | 'rejected_semantic'; -interface FixtureCase { +export interface FixtureCase { label: string; mode: FixtureMode; slot: FixtureSlot; @@ -47,57 +47,23 @@ interface FixtureGroup { type FixtureFile = Partial>; -interface Outcome { - accept: boolean; - messages: string[]; -} - interface ListenerError { message: string; isWarning: boolean; } +export interface Outcome { + accept: boolean; + messages: string[]; +} + const GRAMMAR_FILE_BY_MODE: Record = { bucket_definitions: fileURLToPath(new URL('../../../grammar/1-bucket-definitions.ebnf', import.meta.url)), sync_streams_alpha: fileURLToPath(new URL('../../../grammar/2-sync-streams-alpha.ebnf', import.meta.url)), new_compiler: fileURLToPath(new URL('../../../grammar/3-sync-streams-compiler.ebnf', import.meta.url)) }; -const fixtures: FixtureCase[] = [ - ...loadFixtureFile('fixtures/bucket_definitions.yaml', 'bucket_definitions'), - ...loadFixtureFile('fixtures/sync_streams_alpha.yaml', 'sync_streams_alpha'), - ...loadFixtureFile('fixtures/new_compiler.yaml', 'new_compiler') -]; - -describe('grammar parity fixtures', () => { - test.each(fixtures)('parser contract: $mode/$slot/$kind/$label', (fixture) => { - const outcome = runParser(fixture); - assertParserExpectation(fixture, outcome); - }); - - test.each(fixtures)('grammar contract: $mode/$slot/$kind/$label', (fixture) => { - const outcome = runGrammarChecker(fixture); - assertGrammarExpectation(fixture, outcome); - }); - - test.each(fixtures)('parser/grammar matrix: $mode/$slot/$kind/$label', (fixture) => { - const parserOutcome = runParser(fixture); - const grammarOutcome = runGrammarChecker(fixture); - - expect( - { - parser: parserOutcome.accept, - grammar: grammarOutcome.accept - }, - `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` - ).toEqual({ - parser: fixture.parserOk, - grammar: fixture.grammarOk - }); - }); -}); - -function loadFixtureFile(relativePath: string, mode: FixtureMode): FixtureCase[] { +export function loadFixtureFile(relativePath: string, mode: FixtureMode): FixtureCase[] { const filePath = fileURLToPath(new URL(relativePath, import.meta.url)); const parsed = parseYaml(readFileSync(filePath, 'utf8')) as FixtureFile; const output: FixtureCase[] = []; @@ -115,42 +81,29 @@ function loadFixtureFile(relativePath: string, mode: FixtureMode): FixtureCase[] return output; } -function pushGroup( - out: FixtureCase[], - mode: FixtureMode, - slot: FixtureSlot, - kind: FixtureKind, - entries: FixtureEntry[] -): void { - entries.forEach((entry, index) => { - const expected = expectedOutcomes(kind); - - out.push({ - mode, - slot, - kind, - sql: entry.sql, - err: entry.err, - params: entry.params, - parserOk: expected.parserOk, - grammarOk: expected.grammarOk, - label: fixtureLabel(entry.sql, index) - }); - }); +export function runParser(fixture: FixtureCase): Outcome { + switch (fixture.mode) { + case 'bucket_definitions': + return runBucketParser(fixture); + case 'sync_streams_alpha': + return runSyncStreamsAlphaParser(fixture); + case 'new_compiler': + return runNewCompilerParser(fixture); + } } -function expectedOutcomes(kind: FixtureKind): { parserOk: boolean; grammarOk: boolean } { - switch (kind) { - case 'accepted': - return { parserOk: true, grammarOk: true }; - case 'rejected_syntax': - return { parserOk: false, grammarOk: false }; - case 'rejected_semantic': - return { parserOk: false, grammarOk: true }; - } +export function runGrammarChecker(fixture: FixtureCase): Outcome { + const grammarPath = GRAMMAR_FILE_BY_MODE[fixture.mode]; + const startRule = grammarStartRule(fixture.mode, fixture.slot); + + const accept = grammarAcceptsSql(grammarPath, startRule, fixture.sql); + return { + accept, + messages: accept ? [] : [`Grammar ${startRule} did not accept SQL.`] + }; } -function assertParserExpectation(fixture: FixtureCase, outcome: Outcome): void { +export function assertParserExpectation(fixture: FixtureCase, outcome: Outcome): void { expect( { mode: fixture.mode, @@ -173,7 +126,7 @@ function assertParserExpectation(fixture: FixtureCase, outcome: Outcome): void { } } -function assertGrammarExpectation(fixture: FixtureCase, outcome: Outcome): void { +export function assertGrammarExpectation(fixture: FixtureCase, outcome: Outcome): void { expect( { mode: fixture.mode, @@ -188,27 +141,51 @@ function assertGrammarExpectation(fixture: FixtureCase, outcome: Outcome): void ).toMatchObject({ actual: fixture.grammarOk }); } -function fixtureLabel(sql: string, index: number): string { - const oneLine = sql.replace(/\s+/g, ' ').trim(); - const clipped = oneLine.length > 60 ? `${oneLine.slice(0, 57)}...` : oneLine; - return `${index + 1}. ${clipped}`; +export function fixtureRef(fixture: FixtureCase): string { + return `${fixture.mode}/${fixture.slot}/${fixture.kind}/${fixture.label}`; } -function fixtureRef(fixture: FixtureCase): string { - return `${fixture.mode}/${fixture.slot}/${fixture.kind}/${fixture.label}`; +function pushGroup( + out: FixtureCase[], + mode: FixtureMode, + slot: FixtureSlot, + kind: FixtureKind, + entries: FixtureEntry[] +): void { + entries.forEach((entry, index) => { + const expected = expectedOutcomes(kind); + + out.push({ + mode, + slot, + kind, + sql: entry.sql, + err: entry.err, + params: entry.params, + parserOk: expected.parserOk, + grammarOk: expected.grammarOk, + label: fixtureLabel(entry.sql, index) + }); + }); } -function runParser(fixture: FixtureCase): Outcome { - switch (fixture.mode) { - case 'bucket_definitions': - return runBucketParser(fixture); - case 'sync_streams_alpha': - return runSyncStreamsAlphaParser(fixture); - case 'new_compiler': - return runNewCompilerParser(fixture); +function expectedOutcomes(kind: FixtureKind): { parserOk: boolean; grammarOk: boolean } { + switch (kind) { + case 'accepted': + return { parserOk: true, grammarOk: true }; + case 'rejected_syntax': + return { parserOk: false, grammarOk: false }; + case 'rejected_semantic': + return { parserOk: false, grammarOk: true }; } } +function fixtureLabel(sql: string, index: number): string { + const oneLine = sql.replace(/\s+/g, ' ').trim(); + const clipped = oneLine.length > 60 ? `${oneLine.slice(0, 57)}...` : oneLine; + return `${index + 1}. ${clipped}`; +} + function runBucketParser(fixture: FixtureCase): Outcome { const slot = fixture.slot as BucketSlot; @@ -239,8 +216,8 @@ function runBucketParser(fixture: FixtureCase): Outcome { const messages = hardParserMessages(parsed.errors); return { accept: messages.length === 0, messages }; - } catch (e) { - return rejectFromException(e); + } catch (error) { + return rejectFromException(error); } } @@ -254,8 +231,8 @@ function runSyncStreamsAlphaParser(fixture: FixtureCase): Outcome { const messages = hardParserMessages(errors); return { accept: messages.length === 0, messages }; - } catch (e) { - return rejectFromException(e); + } catch (error) { + return rejectFromException(error); } } @@ -281,24 +258,13 @@ function runNewCompilerParser(fixture: FixtureCase): Outcome { stream.finish(); } - const messages = errors.filter((e) => !e.isWarning).map((e) => e.message); + const messages = errors.filter((error) => !error.isWarning).map((error) => error.message); return { accept: messages.length === 0, messages }; - } catch (e) { - return rejectFromException(e); + } catch (error) { + return rejectFromException(error); } } -function runGrammarChecker(fixture: FixtureCase): Outcome { - const grammarPath = GRAMMAR_FILE_BY_MODE[fixture.mode]; - const startRule = grammarStartRule(fixture.mode, fixture.slot); - - const accept = grammarAcceptsSql(grammarPath, startRule, fixture.sql); - return { - accept, - messages: accept ? [] : [`Grammar ${startRule} did not accept SQL.`] - }; -} - function grammarStartRule(mode: FixtureMode, slot: FixtureSlot): string { if (mode === 'bucket_definitions') { return slot === 'parameters' ? 'ParameterQuery' : 'DataQuery'; diff --git a/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts new file mode 100644 index 000000000..1c173506c --- /dev/null +++ b/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from 'vitest'; +import { + assertGrammarExpectation, + assertParserExpectation, + fixtureRef, + loadFixtureFile, + runGrammarChecker, + runParser +} from './parity_helpers.js'; + +const fixtures = loadFixtureFile('fixtures/sync_streams_alpha.yaml', 'sync_streams_alpha'); + +describe('grammar parity fixtures: sync_streams_alpha', () => { + test.each(fixtures)('parser contract: $slot/$kind/$label', (fixture) => { + const outcome = runParser(fixture); + assertParserExpectation(fixture, outcome); + }); + + test.each(fixtures)('grammar contract: $slot/$kind/$label', (fixture) => { + const outcome = runGrammarChecker(fixture); + assertGrammarExpectation(fixture, outcome); + }); + + test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { + const parserOutcome = runParser(fixture); + const grammarOutcome = runGrammarChecker(fixture); + + expect( + { + parser: parserOutcome.accept, + grammar: grammarOutcome.accept + }, + `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` + ).toEqual({ + parser: fixture.parserOk, + grammar: fixture.grammarOk + }); + }); +}); From d09f51a813610ac162ff0d7030c65723fda1e342 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 11:59:14 +0200 Subject: [PATCH 03/13] Fix some grammar issues. --- .../sync-rules/grammar/1-bucket-definitions.ebnf | 14 +++++++------- .../sync-rules/grammar/2-sync-streams-alpha.ebnf | 8 +++++--- .../grammar/3-sync-streams-compiler.ebnf | 16 ++++++++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/packages/sync-rules/grammar/1-bucket-definitions.ebnf b/packages/sync-rules/grammar/1-bucket-definitions.ebnf index 19081c9a8..c5a01bc50 100644 --- a/packages/sync-rules/grammar/1-bucket-definitions.ebnf +++ b/packages/sync-rules/grammar/1-bucket-definitions.ebnf @@ -21,14 +21,14 @@ BucketDefinitionSql ::= ParameterQuery | DataQuery /* Parameter-query SQL forms used under bucket_definitions..parameters */ ParameterQuery ::= TableValuedParameterQuery | TableParameterQuery | StaticParameterQuery -StaticParameterQuery ::= "SELECT" SelectList ("WHERE" ParameterExpr)? +StaticParameterQuery ::= "SELECT" SelectList ("WHERE" MatchExpr)? TableParameterQuery ::= "SELECT" SelectList "FROM" TableRef ("WHERE" MatchExpr)? /* Only json_each(...) table-valued FROM is supported in this mode. */ -TableValuedParameterQuery ::= "SELECT" SelectList "FROM" JsonEachCall Alias? ("WHERE" ParameterExpr)? +TableValuedParameterQuery ::= "SELECT" SelectList "FROM" JsonEachCall Alias? ("WHERE" MatchExpr)? -JsonEachCall ::= "JSON_EACH" "(" ParameterExpr ")" +JsonEachCall ::= "JSON_EACH" "(" ScalarExpr ")" /* Data-query SQL forms used under bucket_definitions..data[] */ DataQuery ::= "SELECT" DataSelectList "FROM" TableRef ("WHERE" DataMatchExpr)? @@ -45,8 +45,6 @@ Alias ::= "AS" Identifier TableRef ::= Identifier ("." Identifier)? -ParameterExpr ::= ScalarExpr - DataMatchExpr ::= MatchExpr MatchExpr ::= OrExpr @@ -56,7 +54,9 @@ OrExpr ::= AndExpr ("OR" AndExpr)* AndExpr ::= UnaryExpr ("AND" UnaryExpr)* -UnaryExpr ::= "NOT"? Predicate +UnaryExpr ::= "NOT"? MatchAtom + +MatchAtom ::= Predicate | "(" MatchExpr ")" Predicate ::= ScalarExpr PredicateTail | ScalarExpr @@ -69,7 +69,7 @@ ValueTerm ::= PrimaryTerm MemberSuffix* PrimaryTerm ::= Literal | CastExpr | FunctionCall | Reference | "(" ScalarExpr ")" -MemberSuffix ::= ("->" | "->>") (StringLiteral | IntegerLiteral) +MemberSuffix ::= ("->>" | "->") (StringLiteral | IntegerLiteral) CastExpr ::= "CAST" "(" ScalarExpr "AS" CastType ")" diff --git a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf index 4e84ed478..43ba69e3b 100644 --- a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf +++ b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf @@ -31,7 +31,9 @@ OrExpr ::= AndExpr ("OR" AndExpr)* AndExpr ::= UnaryExpr ("AND" UnaryExpr)* -UnaryExpr ::= "NOT"? StreamPredicate +UnaryExpr ::= "NOT"? StreamMatchAtom + +StreamMatchAtom ::= StreamPredicate | "(" StreamWhereExpr ")" StreamPredicate ::= ScalarExpr StreamPredicateTail | ScalarExpr @@ -51,7 +53,7 @@ SubqueryAndExpr ::= SubqueryPredicate ("AND" SubqueryPredicate)* SubqueryPredicate ::= ScalarExpr SubqueryPredicateTail | ScalarExpr -SubqueryPredicateTail ::= "=" ScalarExpr | "IN" ScalarExpr | "&&" ScalarExpr +SubqueryPredicateTail ::= "=" ScalarExpr | "IN" ScalarExpr | "&&" ScalarExpr | "IS" "NOT"? "NULL" /* Scalar expression subset (sql_filters/sql_functions) */ ScalarExpr ::= ValueTerm (BinaryOp ValueTerm)* @@ -60,7 +62,7 @@ ValueTerm ::= PrimaryTerm MemberSuffix* PrimaryTerm ::= Literal | CastExpr | FunctionCall | Reference | "(" ScalarExpr ")" -MemberSuffix ::= ("->" | "->>") (StringLiteral | IntegerLiteral) +MemberSuffix ::= ("->>" | "->") (StringLiteral | IntegerLiteral) CastExpr ::= "CAST" "(" ScalarExpr "AS" CastType ")" diff --git a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf index 4c645568f..1b4c6cdfd 100644 --- a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf +++ b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf @@ -28,7 +28,7 @@ ResultColumn ::= "*" | Reference "." "*" | ScalarExpr Alias? FromContinuation ::= "," FromSource | JoinClause -FromSource ::= TableSource | TableValuedSource | SubquerySource +FromSource ::= TableValuedSource | SubquerySource | TableSource /* Table and table-valued sources may be unaliased or AS-aliased */ TableSource ::= TableRef Alias | TableRef @@ -49,7 +49,9 @@ OrExpr ::= AndExpr ("OR" AndExpr)* AndExpr ::= UnaryExpr ("AND" UnaryExpr)* -UnaryExpr ::= "NOT"? Predicate +UnaryExpr ::= "NOT"? WhereAtom + +WhereAtom ::= Predicate | "(" WhereExpr ")" Predicate ::= ScalarExpr PredicateTail | ScalarExpr @@ -77,12 +79,18 @@ ValueTerm ::= PrimaryTerm MemberSuffix* PrimaryTerm ::= Literal | CaseExpr | CastExpr | FunctionCall | Reference | "(" ScalarExpr ")" -MemberSuffix ::= ("->" | "->>") (StringLiteral | IntegerLiteral) +MemberSuffix ::= ("->>" | "->") (StringLiteral | IntegerLiteral) CastExpr ::= "CAST" "(" ScalarExpr "AS" CastType ")" /* CASE expression subset supported by compiler parser */ -CaseExpr ::= "CASE" ScalarExpr? "WHEN" ScalarExpr "THEN" ScalarExpr ("WHEN" ScalarExpr "THEN" ScalarExpr)* ("ELSE" ScalarExpr)? "END" +CaseExpr ::= SearchedCaseExpr | SimpleCaseExpr + +SearchedCaseExpr ::= "CASE" "WHEN" CaseCondition "THEN" ScalarExpr ("WHEN" CaseCondition "THEN" ScalarExpr)* ("ELSE" ScalarExpr)? "END" + +SimpleCaseExpr ::= "CASE" ScalarExpr "WHEN" ScalarExpr "THEN" ScalarExpr ("WHEN" ScalarExpr "THEN" ScalarExpr)* ("ELSE" ScalarExpr)? "END" + +CaseCondition ::= OrExpr FunctionCall ::= (Identifier ".")? Identifier "(" ArgList? ")" From 25af72786efec5020503d27744ef6c7ac25a7212 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 12:12:09 +0200 Subject: [PATCH 04/13] More test fixes. --- .../bucket_definitions_parity.test.ts | 16 ++----- .../fixtures/bucket_definitions.yaml | 20 ++++++-- .../grammar_parity/fixtures/new_compiler.yaml | 15 ++++-- .../fixtures/sync_streams_alpha.yaml | 15 ++++-- .../new_compiler_parity.test.ts | 16 ++----- .../test/src/grammar_parity/parity_helpers.ts | 48 ++++++++++++++----- .../sync_streams_alpha_parity.test.ts | 16 ++----- 7 files changed, 84 insertions(+), 62 deletions(-) diff --git a/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts index c8cd104dc..caad23626 100644 --- a/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts +++ b/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts @@ -1,8 +1,8 @@ -import { describe, expect, test } from 'vitest'; +import { describe, test } from 'vitest'; import { assertGrammarExpectation, + assertMatrixExpectation, assertParserExpectation, - fixtureRef, loadFixtureFile, runGrammarChecker, runParser @@ -24,16 +24,6 @@ describe('grammar parity fixtures: bucket_definitions', () => { test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { const parserOutcome = runParser(fixture); const grammarOutcome = runGrammarChecker(fixture); - - expect( - { - parser: parserOutcome.accept, - grammar: grammarOutcome.accept - }, - `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` - ).toEqual({ - parser: fixture.parserOk, - grammar: fixture.grammarOk - }); + assertMatrixExpectation(fixture, parserOutcome, grammarOutcome); }); }); diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml index 5d099e510..de2a53717 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml @@ -20,7 +20,6 @@ parameters: - sql: SELECT CAST(request.parameter('limit') AS integer) AS limit_value - sql: SELECT token_parameters.user_id AS user_id WHERE request.user_id() IS NOT NULL - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id AND token_parameters.user_id IS NOT NULL - - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id OR users.id = request.user_id() - sql: SELECT CAST(request.parameters() ->> 'org_id' AS text) AS org_id rejected_syntax: @@ -38,6 +37,11 @@ parameters: - sql: SELECT token_parameters.user_id AS user_id LIMIT 1 err: LIMIT + rejected_semantic: + # parser: Left and right sides of OR must use the same parameters, or split into separate queries. [{"key":"token_parameters.user_id","expands":false}] != [{"key":"request.user_id()","expands":false}] + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id OR users.id = request.user_id() + err: same parameters + data: accepted: - sql: SELECT id FROM assets @@ -51,8 +55,6 @@ data: params: [user_id] - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND id IS NOT NULL params: [user_id] - - sql: SELECT id FROM assets WHERE NOT (owner_id != bucket.user_id) - params: [user_id] - sql: SELECT id, name AS asset_name FROM assets WHERE owner_id = bucket.user_id params: [user_id] - sql: SELECT id FROM test_schema.assets WHERE owner_id = bucket.user_id @@ -65,8 +67,6 @@ data: params: [user_id] - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND id = id params: [user_id] - - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (bucket.user_id IS NOT NULL) - params: [user_id] - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (owner_id = bucket.user_id) params: [user_id] - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (owner_id IS NOT NULL) @@ -93,3 +93,13 @@ data: - sql: SELECT id FROM assets params: [user_id] err: cover all bucket parameters + + # parser: Cannot use bucket parameters in expressions + - sql: SELECT id FROM assets WHERE NOT (owner_id != bucket.user_id) + params: [user_id] + err: bucket parameters in expressions + + # parser: Cannot use bucket parameters in expressions + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (bucket.user_id IS NOT NULL) + params: [user_id] + err: bucket parameters in expressions diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml index 171c41a36..1d7170460 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml @@ -14,19 +14,16 @@ query: - sql: SELECT * FROM users WHERE id = id - sql: SELECT * FROM users WHERE id IN (SELECT id FROM users) - sql: SELECT * FROM users WHERE id IN (SELECT id FROM users WHERE id IS NOT NULL) - - sql: SELECT * FROM users WHERE id NOT IN (SELECT id FROM users WHERE id IS NOT NULL) - sql: SELECT * FROM users WHERE id BETWEEN 1 AND 10 - sql: SELECT * FROM users WHERE id NOT BETWEEN 1 AND 10 - sql: SELECT * FROM users WHERE auth.user_id() IS NOT NULL - sql: SELECT * FROM users WHERE auth.parameter('tenant') IS NOT NULL - sql: SELECT * FROM users WHERE subscription.parameter('plan') IS NOT NULL - sql: SELECT * FROM users WHERE connection.parameter('id') IS NOT NULL - - sql: SELECT j.value AS value FROM json_each(auth.parameters()) AS j - sql: SELECT u.id FROM users AS u WHERE (u.id = u.id) - sql: SELECT u.id FROM users AS u WHERE (u.id = auth.parameter('id')) - sql: SELECT u.id FROM users AS u WHERE (u.id = connection.parameter('id')) - sql: SELECT u.id FROM users AS u WHERE (u.id = subscription.parameter('id')) - - sql: SELECT u.id, o.id FROM users AS u JOIN orgs AS o ON u.org_id = o.id - sql: SELECT u.id FROM users AS u WHERE u.id IN (SELECT id FROM users AS x) - sql: SELECT * FROM users WHERE id && (SELECT id FROM users WHERE id IS NOT NULL) @@ -52,6 +49,18 @@ query: err: SELECT rejected_semantic: + # parser: This expression already references 'users', so it can't also reference data from this row unless the two are compared with an equals operator. | This filter is unrelated to the request or the table being synced, and not supported. + - sql: SELECT * FROM users WHERE id NOT IN (SELECT id FROM users WHERE id IS NOT NULL) + err: can't also reference data from this row + + # parser: Sync streams can only select from actual tables | Must have a result column selecting from a table + - sql: SELECT j.value AS value FROM json_each(auth.parameters()) AS j + err: select from actual tables + + # parser: Sync streams can only select from a single table, and this one already selects from 'users'. + - sql: SELECT u.id, o.id FROM users AS u JOIN orgs AS o ON u.org_id = o.id + err: select from a single table + # parser: This attempts to sync a connection parameter. Only values from the source database can be synced. - sql: SELECT u.*, auth.parameter('x') FROM users AS u err: connection parameter diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml index d88e0e6ec..d90dd35e4 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml @@ -5,15 +5,12 @@ query: - sql: SELECT id FROM comments - sql: SELECT comments.id FROM comments - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND id IS NOT NULL) - - sql: SELECT * FROM comments WHERE issue_id NOT IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) - sql: SELECT * FROM comments WHERE auth.user_id() IS NOT NULL - - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues) - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.parameter('owner_id')) - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = connection.parameter('owner_id')) - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = subscription.parameter('owner_id')) - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND owner_id = auth.user_id()) - sql: SELECT * FROM comments WHERE issue_id = issue_id - - sql: SELECT * FROM comments WHERE NOT (issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id())) - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND name = 'x') - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() AND id = id) - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM test_schema.issues WHERE owner_id = auth.user_id()) @@ -45,6 +42,18 @@ query: err: OR rejected_semantic: + # parser: Negations are not allowed here + - sql: SELECT * FROM comments WHERE issue_id NOT IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) + err: negations are not allowed + + # parser: Unsupported subquery without parameter, must depend on request parameters + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues) + err: unsupported subquery without parameter + + # parser: Negations are not allowed here + - sql: SELECT * FROM comments WHERE NOT (issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id())) + err: negations are not allowed + # parser: For IN subqueries, the left operand must either depend on the row to sync or stream parameters. - sql: SELECT * FROM comments WHERE 'static' IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) err: left operand diff --git a/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts index b5499e3ca..222f30c95 100644 --- a/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts +++ b/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts @@ -1,8 +1,8 @@ -import { describe, expect, test } from 'vitest'; +import { describe, test } from 'vitest'; import { assertGrammarExpectation, + assertMatrixExpectation, assertParserExpectation, - fixtureRef, loadFixtureFile, runGrammarChecker, runParser @@ -24,16 +24,6 @@ describe('grammar parity fixtures: new_compiler', () => { test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { const parserOutcome = runParser(fixture); const grammarOutcome = runGrammarChecker(fixture); - - expect( - { - parser: parserOutcome.accept, - grammar: grammarOutcome.accept - }, - `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` - ).toEqual({ - parser: fixture.parserOk, - grammar: fixture.grammarOk - }); + assertMatrixExpectation(fixture, parserOutcome, grammarOutcome); }); }); diff --git a/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts index bc248b979..af98ed6c1 100644 --- a/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts +++ b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts @@ -104,18 +104,17 @@ export function runGrammarChecker(fixture: FixtureCase): Outcome { } export function assertParserExpectation(fixture: FixtureCase, outcome: Outcome): void { - expect( - { - mode: fixture.mode, - slot: fixture.slot, - kind: fixture.kind, - sql: fixture.sql, - expected: fixture.parserOk, - actual: outcome.accept, - messages: outcome.messages - }, - `Parser expectation mismatch for ${fixtureRef(fixture)}` - ).toMatchObject({ actual: fixture.parserOk }); + if (outcome.accept !== fixture.parserOk) { + expect.fail( + [ + `Parser expectation mismatch for ${fixtureRef(fixture)}`, + `Expected parser acceptance: ${fixture.parserOk}`, + `Actual parser acceptance: ${outcome.accept}`, + `Parser messages: ${formatMessages(outcome.messages)}`, + `SQL: ${fixture.sql}` + ].join('\n') + ); + } if (!fixture.parserOk && fixture.err) { const needle = fixture.err.toLowerCase(); @@ -141,6 +140,23 @@ export function assertGrammarExpectation(fixture: FixtureCase, outcome: Outcome) ).toMatchObject({ actual: fixture.grammarOk }); } +export function assertMatrixExpectation(fixture: FixtureCase, parserOutcome: Outcome, grammarOutcome: Outcome): void { + if (parserOutcome.accept === fixture.parserOk && grammarOutcome.accept === fixture.grammarOk) { + return; + } + + expect.fail( + [ + `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}`, + `Expected matrix: parser=${fixture.parserOk}, grammar=${fixture.grammarOk}`, + `Actual matrix: parser=${parserOutcome.accept}, grammar=${grammarOutcome.accept}`, + `Parser messages: ${formatMessages(parserOutcome.messages)}`, + `Grammar messages: ${formatMessages(grammarOutcome.messages)}`, + `SQL: ${fixture.sql}` + ].join('\n') + ); +} + export function fixtureRef(fixture: FixtureCase): string { return `${fixture.mode}/${fixture.slot}/${fixture.kind}/${fixture.label}`; } @@ -288,3 +304,11 @@ function rejectFromException(error: unknown): Outcome { return { accept: false, messages: [String(error)] }; } + +function formatMessages(messages: string[]): string { + if (messages.length === 0) { + return ''; + } + + return messages.join(' | '); +} diff --git a/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts index 1c173506c..839c0a96f 100644 --- a/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts +++ b/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts @@ -1,8 +1,8 @@ -import { describe, expect, test } from 'vitest'; +import { describe, test } from 'vitest'; import { assertGrammarExpectation, + assertMatrixExpectation, assertParserExpectation, - fixtureRef, loadFixtureFile, runGrammarChecker, runParser @@ -24,16 +24,6 @@ describe('grammar parity fixtures: sync_streams_alpha', () => { test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { const parserOutcome = runParser(fixture); const grammarOutcome = runGrammarChecker(fixture); - - expect( - { - parser: parserOutcome.accept, - grammar: grammarOutcome.accept - }, - `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}` - ).toEqual({ - parser: fixture.parserOk, - grammar: fixture.grammarOk - }); + assertMatrixExpectation(fixture, parserOutcome, grammarOutcome); }); }); From e9a15cc7f446370c70c245aa30a451fbe3ac65a7 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 12:21:10 +0200 Subject: [PATCH 05/13] Remove redundant tests. --- .../bucket_definitions_parity.test.ts | 7 ------- .../grammar_parity/new_compiler_parity.test.ts | 7 ------- .../test/src/grammar_parity/parity_helpers.ts | 17 ----------------- .../sync_streams_alpha_parity.test.ts | 7 ------- 4 files changed, 38 deletions(-) diff --git a/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts index caad23626..b0ac2459c 100644 --- a/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts +++ b/packages/sync-rules/test/src/grammar_parity/bucket_definitions_parity.test.ts @@ -1,7 +1,6 @@ import { describe, test } from 'vitest'; import { assertGrammarExpectation, - assertMatrixExpectation, assertParserExpectation, loadFixtureFile, runGrammarChecker, @@ -20,10 +19,4 @@ describe('grammar parity fixtures: bucket_definitions', () => { const outcome = runGrammarChecker(fixture); assertGrammarExpectation(fixture, outcome); }); - - test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { - const parserOutcome = runParser(fixture); - const grammarOutcome = runGrammarChecker(fixture); - assertMatrixExpectation(fixture, parserOutcome, grammarOutcome); - }); }); diff --git a/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts index 222f30c95..f3f1ea569 100644 --- a/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts +++ b/packages/sync-rules/test/src/grammar_parity/new_compiler_parity.test.ts @@ -1,7 +1,6 @@ import { describe, test } from 'vitest'; import { assertGrammarExpectation, - assertMatrixExpectation, assertParserExpectation, loadFixtureFile, runGrammarChecker, @@ -20,10 +19,4 @@ describe('grammar parity fixtures: new_compiler', () => { const outcome = runGrammarChecker(fixture); assertGrammarExpectation(fixture, outcome); }); - - test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { - const parserOutcome = runParser(fixture); - const grammarOutcome = runGrammarChecker(fixture); - assertMatrixExpectation(fixture, parserOutcome, grammarOutcome); - }); }); diff --git a/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts index af98ed6c1..d37c66e83 100644 --- a/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts +++ b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts @@ -140,23 +140,6 @@ export function assertGrammarExpectation(fixture: FixtureCase, outcome: Outcome) ).toMatchObject({ actual: fixture.grammarOk }); } -export function assertMatrixExpectation(fixture: FixtureCase, parserOutcome: Outcome, grammarOutcome: Outcome): void { - if (parserOutcome.accept === fixture.parserOk && grammarOutcome.accept === fixture.grammarOk) { - return; - } - - expect.fail( - [ - `Parser/grammar matrix mismatch for ${fixtureRef(fixture)}`, - `Expected matrix: parser=${fixture.parserOk}, grammar=${fixture.grammarOk}`, - `Actual matrix: parser=${parserOutcome.accept}, grammar=${grammarOutcome.accept}`, - `Parser messages: ${formatMessages(parserOutcome.messages)}`, - `Grammar messages: ${formatMessages(grammarOutcome.messages)}`, - `SQL: ${fixture.sql}` - ].join('\n') - ); -} - export function fixtureRef(fixture: FixtureCase): string { return `${fixture.mode}/${fixture.slot}/${fixture.kind}/${fixture.label}`; } diff --git a/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts b/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts index 839c0a96f..d435d0c17 100644 --- a/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts +++ b/packages/sync-rules/test/src/grammar_parity/sync_streams_alpha_parity.test.ts @@ -1,7 +1,6 @@ import { describe, test } from 'vitest'; import { assertGrammarExpectation, - assertMatrixExpectation, assertParserExpectation, loadFixtureFile, runGrammarChecker, @@ -20,10 +19,4 @@ describe('grammar parity fixtures: sync_streams_alpha', () => { const outcome = runGrammarChecker(fixture); assertGrammarExpectation(fixture, outcome); }); - - test.each(fixtures)('parser/grammar matrix: $slot/$kind/$label', (fixture) => { - const parserOutcome = runParser(fixture); - const grammarOutcome = runGrammarChecker(fixture); - assertMatrixExpectation(fixture, parserOutcome, grammarOutcome); - }); }); From 71b1e7d7ef0850a5c3a1dea86fbd1ac60c8158c6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 12:21:40 +0200 Subject: [PATCH 06/13] Fix issue with BETWEEN. --- packages/sync-rules/src/compiler/sqlite.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/sync-rules/src/compiler/sqlite.ts b/packages/sync-rules/src/compiler/sqlite.ts index b4ce74812..d3d36f1b5 100644 --- a/packages/sync-rules/src/compiler/sqlite.ts +++ b/packages/sync-rules/src/compiler/sqlite.ts @@ -320,6 +320,7 @@ export class PostgresToSqlite { low: this.translateNodeWithLocation(expr.lo), high: this.translateNodeWithLocation(expr.hi) }; + this.options.locations.sourceForNode.set(between, expr); return expr.op === 'BETWEEN' ? between : { type: 'unary', operator: 'not', operand: between }; } From 4b826559f7dc574cd659093aae26cbc40415d91a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 12:28:50 +0200 Subject: [PATCH 07/13] Clarify string literals. --- packages/sync-rules/grammar/1-bucket-definitions.ebnf | 2 +- packages/sync-rules/grammar/2-sync-streams-alpha.ebnf | 2 +- packages/sync-rules/grammar/3-sync-streams-compiler.ebnf | 2 +- .../test/src/grammar_parity/fixtures/bucket_definitions.yaml | 3 +++ .../test/src/grammar_parity/fixtures/new_compiler.yaml | 3 +++ .../test/src/grammar_parity/fixtures/sync_streams_alpha.yaml | 3 +++ 6 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/sync-rules/grammar/1-bucket-definitions.ebnf b/packages/sync-rules/grammar/1-bucket-definitions.ebnf index c5a01bc50..31a03a8ad 100644 --- a/packages/sync-rules/grammar/1-bucket-definitions.ebnf +++ b/packages/sync-rules/grammar/1-bucket-definitions.ebnf @@ -88,7 +88,7 @@ Identifier ::= [A-Z_][A-Z_0-9]* Literal ::= StringLiteral | NumericLiteral | "TRUE" | "FALSE" | "NULL" -StringLiteral ::= '"' ([#x20-#x21] | [#x23-#x5B] | [#x5D-#x7E])* '"' | "'" ([#x20-#x26] | [#x28-#x7E])* "'" +StringLiteral ::= "'" ([#x20-#x26] | [#x28-#x7E])* "'" IntegerLiteral ::= [0-9]+ diff --git a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf index 43ba69e3b..9d9a21d31 100644 --- a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf +++ b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf @@ -85,7 +85,7 @@ Identifier ::= [A-Z_][A-Z_0-9]* Literal ::= StringLiteral | NumericLiteral | "TRUE" | "FALSE" | "NULL" -StringLiteral ::= '"' ([#x20-#x21] | [#x23-#x5B] | [#x5D-#x7E])* '"' | "'" ([#x20-#x26] | [#x28-#x7E])* "'" +StringLiteral ::= "'" ([#x20-#x26] | [#x28-#x7E])* "'" IntegerLiteral ::= [0-9]+ diff --git a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf index 1b4c6cdfd..4f99b8472 100644 --- a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf +++ b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf @@ -113,7 +113,7 @@ Identifier ::= [A-Z_][A-Z_0-9]* Literal ::= StringLiteral | NumericLiteral | "TRUE" | "FALSE" | "NULL" -StringLiteral ::= '"' ([#x20-#x21] | [#x23-#x5B] | [#x5D-#x7E])* '"' | "'" ([#x20-#x26] | [#x28-#x7E])* "'" +StringLiteral ::= "'" ([#x20-#x26] | [#x28-#x7E])* "'" IntegerLiteral ::= [0-9]+ diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml index de2a53717..6a325db80 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml @@ -28,6 +28,9 @@ parameters: - sql: SELECT u.org_id AS org_id FROM users u WHERE u.id = token_parameters.user_id err: alias + # double quotes denote identifiers, not string literals + - sql: SELECT request.parameter("tenant_id") AS tenant_id + # only json_each(...) table-valued functions are supported # parser: Table-valued function generate_series is not defined. - sql: SELECT x FROM generate_series(1, 3) AS x diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml index 1d7170460..40c15822c 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml @@ -28,6 +28,9 @@ query: - sql: SELECT * FROM users WHERE id && (SELECT id FROM users WHERE id IS NOT NULL) rejected_syntax: + # double quotes denote identifiers, not string literals + - sql: SELECT * FROM users WHERE id = auth.parameter("id") + # parser: FULL JOIN is not supported - sql: SELECT u.* FROM users u FULL JOIN orgs o ON u.org_id = o.id err: JOIN diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml index d90dd35e4..020b6a7db 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml @@ -21,6 +21,9 @@ query: - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND id = id rejected_syntax: + # double quotes denote identifiers, not string literals + - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.parameter("owner_id")) + # parser: GROUP BY is not supported - sql: SELECT id FROM comments GROUP BY id err: GROUP BY From 3ff46ca24ea7e24499acb7fa06fc5e3a5ab8f59f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 12:34:48 +0200 Subject: [PATCH 08/13] Move WS definitions to tests. --- packages/sync-rules/grammar/1-bucket-definitions.ebnf | 3 --- packages/sync-rules/grammar/2-sync-streams-alpha.ebnf | 3 --- packages/sync-rules/grammar/3-sync-streams-compiler.ebnf | 3 --- .../sync-rules/test/src/grammar_parity/generated_grammar.ts | 5 ++++- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/sync-rules/grammar/1-bucket-definitions.ebnf b/packages/sync-rules/grammar/1-bucket-definitions.ebnf index 31a03a8ad..67b4e4530 100644 --- a/packages/sync-rules/grammar/1-bucket-definitions.ebnf +++ b/packages/sync-rules/grammar/1-bucket-definitions.ebnf @@ -93,6 +93,3 @@ StringLiteral ::= "'" ([#x20-#x26] | [#x28-#x7E])* "'" IntegerLiteral ::= [0-9]+ NumericLiteral ::= [0-9]+ ("." [0-9]+)? - -/* Explicit whitespace rule used by the runtime parser adapter. */ -WS ::= [#x20#x09#x0A#x0D]+ diff --git a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf index 9d9a21d31..920459874 100644 --- a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf +++ b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf @@ -90,6 +90,3 @@ StringLiteral ::= "'" ([#x20-#x26] | [#x28-#x7E])* "'" IntegerLiteral ::= [0-9]+ NumericLiteral ::= [0-9]+ ("." [0-9]+)? - -/* Explicit whitespace rule used by the runtime parser adapter. */ -WS ::= [#x20#x09#x0A#x0D]+ diff --git a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf index 4f99b8472..dd07f119b 100644 --- a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf +++ b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf @@ -118,6 +118,3 @@ StringLiteral ::= "'" ([#x20-#x26] | [#x28-#x7E])* "'" IntegerLiteral ::= [0-9]+ NumericLiteral ::= [0-9]+ ("." [0-9]+)? - -/* Explicit whitespace rule used by the runtime parser adapter. */ -WS ::= [#x20#x09#x0A#x0D]+ diff --git a/packages/sync-rules/test/src/grammar_parity/generated_grammar.ts b/packages/sync-rules/test/src/grammar_parity/generated_grammar.ts index cdbbc3f3f..ddc8f6e55 100644 --- a/packages/sync-rules/test/src/grammar_parity/generated_grammar.ts +++ b/packages/sync-rules/test/src/grammar_parity/generated_grammar.ts @@ -30,7 +30,10 @@ function getOrCreateParser(grammarFilePath: string, startRule: string): GrammarP return cached; } - const source = `${readFileSync(grammarFilePath, 'utf8')}\nENTRY ::= ${startRule} EOF\n`; + const grammarSource = readFileSync(grammarFilePath, 'utf8'); + const hasExplicitWs = /\bWS\s*::=/.test(grammarSource); + const injectedWs = hasExplicitWs ? '' : '\nWS ::= [#x20#x09#x0A#x0D]+\n'; + const source = `${grammarSource}${injectedWs}\nENTRY ::= ${startRule} EOF\n`; const parser = new Grammars.W3C.Parser(source); for (const rule of (parser as any).grammarRules ?? []) { if ( From 22f3999cf861f8ab4267fe9aa280a7518f9ff1f6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 13:04:28 +0200 Subject: [PATCH 09/13] Tweak notes. --- packages/sync-rules/grammar/1-bucket-definitions.ebnf | 11 +---------- packages/sync-rules/grammar/2-sync-streams-alpha.ebnf | 6 ------ .../sync-rules/grammar/3-sync-streams-compiler.ebnf | 8 +------- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/packages/sync-rules/grammar/1-bucket-definitions.ebnf b/packages/sync-rules/grammar/1-bucket-definitions.ebnf index 67b4e4530..a9b2b42ad 100644 --- a/packages/sync-rules/grammar/1-bucket-definitions.ebnf +++ b/packages/sync-rules/grammar/1-bucket-definitions.ebnf @@ -1,19 +1,10 @@ /* - PowerSync sync-rules SQL grammar: bucket_definitions mode (legacy parser path). + PowerSync sync-rules SQL grammar: bucket_definitions mode SQL appears in YAML at: - bucket_definitions..parameters - string or array of strings - bucket_definitions..data[] - - Source-of-truth files: - - src/SqlBucketDescriptor.ts - - src/SqlParameterQuery.ts - - src/StaticSqlParameterQuery.ts - - src/TableValuedFunctionSqlParameterQuery.ts - - src/SqlDataQuery.ts - - src/sql_filters.ts - - src/sql_support.ts */ BucketDefinitionSql ::= ParameterQuery | DataQuery diff --git a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf index 920459874..296cd6eaf 100644 --- a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf +++ b/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf @@ -7,12 +7,6 @@ Notes: - This is the sync streams alpha SQL parser path. - YAML keys `with` and `queries` are not supported in this mode. - - Source-of-truth files: - - src/streams/from_sql.ts - - src/streams/functions.ts - - src/sql_filters.ts - - src/sql_support.ts */ SyncStreamsAlphaSql ::= SyncStreamsAlphaQuery diff --git a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf index dd07f119b..5902af5ab 100644 --- a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf +++ b/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf @@ -7,13 +7,7 @@ - streams..with. Notes: - - This grammar is used when config.sync_config_compiler = true (and edition >= 2). - - In this mode, bucket_definitions are rejected. - - Source-of-truth files: - - src/compiler/compiler.ts - - src/compiler/parser.ts - - src/compiler/sqlite.ts + - This grammar is used when config.sync_config_compiler = true (and edition >= 2) */ SyncStreamsCompilerSql ::= CompilerStreamQuery | CompilerSubquery | CompilerCteSubquery From 923f82f1fad6b6f119870d3dacac0982f7ee1dc0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 13:07:32 +0200 Subject: [PATCH 10/13] Rename grammar files. --- .../{1-bucket-definitions.ebnf => bucket-definitions.ebnf} | 0 .../{2-sync-streams-alpha.ebnf => sync-streams-alpha.ebnf} | 0 ...ync-streams-compiler.ebnf => sync-streams-compiler.ebnf} | 0 .../sync-rules/test/src/grammar_parity/parity_helpers.ts | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/sync-rules/grammar/{1-bucket-definitions.ebnf => bucket-definitions.ebnf} (100%) rename packages/sync-rules/grammar/{2-sync-streams-alpha.ebnf => sync-streams-alpha.ebnf} (100%) rename packages/sync-rules/grammar/{3-sync-streams-compiler.ebnf => sync-streams-compiler.ebnf} (100%) diff --git a/packages/sync-rules/grammar/1-bucket-definitions.ebnf b/packages/sync-rules/grammar/bucket-definitions.ebnf similarity index 100% rename from packages/sync-rules/grammar/1-bucket-definitions.ebnf rename to packages/sync-rules/grammar/bucket-definitions.ebnf diff --git a/packages/sync-rules/grammar/2-sync-streams-alpha.ebnf b/packages/sync-rules/grammar/sync-streams-alpha.ebnf similarity index 100% rename from packages/sync-rules/grammar/2-sync-streams-alpha.ebnf rename to packages/sync-rules/grammar/sync-streams-alpha.ebnf diff --git a/packages/sync-rules/grammar/3-sync-streams-compiler.ebnf b/packages/sync-rules/grammar/sync-streams-compiler.ebnf similarity index 100% rename from packages/sync-rules/grammar/3-sync-streams-compiler.ebnf rename to packages/sync-rules/grammar/sync-streams-compiler.ebnf diff --git a/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts index d37c66e83..3afffce0f 100644 --- a/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts +++ b/packages/sync-rules/test/src/grammar_parity/parity_helpers.ts @@ -58,9 +58,9 @@ export interface Outcome { } const GRAMMAR_FILE_BY_MODE: Record = { - bucket_definitions: fileURLToPath(new URL('../../../grammar/1-bucket-definitions.ebnf', import.meta.url)), - sync_streams_alpha: fileURLToPath(new URL('../../../grammar/2-sync-streams-alpha.ebnf', import.meta.url)), - new_compiler: fileURLToPath(new URL('../../../grammar/3-sync-streams-compiler.ebnf', import.meta.url)) + bucket_definitions: fileURLToPath(new URL('../../../grammar/bucket-definitions.ebnf', import.meta.url)), + sync_streams_alpha: fileURLToPath(new URL('../../../grammar/sync-streams-alpha.ebnf', import.meta.url)), + new_compiler: fileURLToPath(new URL('../../../grammar/sync-streams-compiler.ebnf', import.meta.url)) }; export function loadFixtureFile(relativePath: string, mode: FixtureMode): FixtureCase[] { From 069279085ebdbc1ad7b42139861ecdce293cb4ea Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 14:22:32 +0200 Subject: [PATCH 11/13] More examples. --- .../fixtures/bucket_definitions.yaml | 28 ++- .../grammar_parity/fixtures/new_compiler.yaml | 32 ++++ .../fixtures/sync_streams_alpha.yaml | 23 +++ where-clause-support.md | 167 ++++++++++++++++++ 4 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 where-clause-support.md diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml index 6a325db80..2159f0466 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/bucket_definitions.yaml @@ -21,6 +21,9 @@ parameters: - sql: SELECT token_parameters.user_id AS user_id WHERE request.user_id() IS NOT NULL - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id AND token_parameters.user_id IS NOT NULL - sql: SELECT CAST(request.parameters() ->> 'org_id' AS text) AS org_id + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = request.user_id() AND users.is_admin = TRUE + - sql: SELECT users.org_id AS org_id FROM users WHERE users.owner_id = request.user_id() OR users.shared_with = request.user_id() + - sql: SELECT users.org_id AS org_id FROM users WHERE NOT users.is_admin = TRUE rejected_syntax: # table aliases are not supported in parameter queries @@ -45,6 +48,16 @@ parameters: - sql: SELECT users.org_id AS org_id FROM users WHERE users.id = token_parameters.user_id OR users.id = request.user_id() err: same parameters + # OR sides reference different parameters + # parser: Left and right sides of OR must use the same parameters, or split into separate queries. [{"key":"request.user_id()","expands":false}] != [{"key":"token_parameters.org_id","expands":false}] + - sql: SELECT users.org_id AS org_id FROM users WHERE users.owner_id = request.user_id() OR users.org_id = token_parameters.org_id + err: same parameters + + # two IN expressions in one AND + # parser: Cannot have multiple array input parameters + - sql: SELECT users.org_id AS org_id FROM users WHERE users.id IN token_parameters.list_ids AND users.org_id IN token_parameters.org_ids + err: multiple array input parameters + data: accepted: - sql: SELECT id FROM assets @@ -78,14 +91,27 @@ data: params: [user_id] - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (id = id) params: [user_id] - - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (owner_id = bucket.user_id OR owner_id = bucket.user_id) + - sql: SELECT id FROM assets WHERE owner_id = bucket.user_id AND (owner_id = bucket.user_id OR other_id = bucket.user_id) params: [user_id] + - sql: SELECT id FROM assets WHERE status = 'active' + - sql: SELECT id FROM assets WHERE deleted_at IS NULL + - sql: SELECT id FROM assets WHERE status != 'archived' + - sql: SELECT id FROM assets WHERE deleted_at IS NOT NULL + - sql: SELECT id FROM assets WHERE category IN '["draft", "hidden"]' rejected_syntax: # parser: Must SELECT from a single table - sql: SELECT assets.id FROM assets JOIN users ON users.id = assets.owner_id err: single table + # NOT IN with fixed value list + # parser: list not supported here + - sql: SELECT id FROM assets WHERE category NOT IN ('draft', 'hidden') + err: list not supported here + - sql: SELECT id FROM assets WHERE category NOT IN '["draft", "hidden"]' + + - sql: SELECT id FROM assets WHERE category IN ('draft', 'hidden') + rejected_semantic: # parser: alias is required - sql: SELECT upper(name) FROM assets diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml index 40c15822c..95544b5ec 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml @@ -26,6 +26,23 @@ query: - sql: SELECT u.id FROM users AS u WHERE (u.id = subscription.parameter('id')) - sql: SELECT u.id FROM users AS u WHERE u.id IN (SELECT id FROM users AS x) - sql: SELECT * FROM users WHERE id && (SELECT id FROM users WHERE id IS NOT NULL) + - sql: SELECT * FROM comments WHERE status = 'active' + - sql: SELECT * FROM comments WHERE deleted_at IS NULL + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() + - sql: SELECT * FROM comments WHERE region = connection.parameter('region') + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() AND org_id = auth.parameter('org_id') + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() AND status = 'active' + - sql: SELECT * FROM comments WHERE list_id = subscription.parameter('list_id') AND list_id IN (SELECT id FROM lists WHERE owner_id = auth.user_id()) + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() OR shared_with = auth.user_id() + - sql: SELECT * FROM comments WHERE status = 'active' AND (owner_id = auth.user_id() OR shared_with = auth.user_id()) + - sql: SELECT * FROM comments WHERE deleted_at IS NOT NULL + - sql: SELECT * FROM comments WHERE category IN '["draft", "hidden"]' + # FIXME: This one should work: + # - sql: SELECT * FROM comments WHERE status != 'archived' + # FIXME: This one should work: + # - sql: SELECT * FROM comments WHERE category NOT IN '["draft", "hidden"]' + # TODO: should we support this? + # - sql: SELECT id FROM assets WHERE category IN ('draft', 'hidden') rejected_syntax: # double quotes denote identifiers, not string literals @@ -51,6 +68,16 @@ query: - sql: DELETE FROM users err: SELECT + # where-clause-support.md example (NOT IN with fixed value list) + # parser: This expression is not supported by PowerSync + - sql: SELECT * FROM comments WHERE category NOT IN ('draft', 'hidden') + err: not supported by PowerSync + + # where-clause-support.md example (NOT IN with parameter source) + # parser: This expression already references 'comments', so it can't also reference data from this row unless the two are compared with an equals operator. + - sql: SELECT * FROM comments WHERE id NOT IN subscription.parameter('excluded_ids') + err: can't also reference data from this row + rejected_semantic: # parser: This expression already references 'users', so it can't also reference data from this row unless the two are compared with an equals operator. | This filter is unrelated to the request or the table being synced, and not supported. - sql: SELECT * FROM users WHERE id NOT IN (SELECT id FROM users WHERE id IS NOT NULL) @@ -68,6 +95,11 @@ query: - sql: SELECT u.*, auth.parameter('x') FROM users AS u err: connection parameter + # where-clause-support.md example (NOT IN with subquery) + # parser: This expression already references 'comments', so it can't also reference data from this row unless the two are compared with an equals operator. | This expression already references row data, so it can't also reference connection parameters unless the two are compared with an equals operator. + - sql: SELECT * FROM comments WHERE issue_id NOT IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) + err: can't also reference data from this row + with: accepted: - sql: SELECT id FROM orgs WHERE owner_id = auth.user_id() diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml index 020b6a7db..b4fb70c21 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/sync_streams_alpha.yaml @@ -19,6 +19,19 @@ query: - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND subscription.parameter('plan') IS NOT NULL - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND auth.parameter('tenant') IS NOT NULL - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) AND id = id + - sql: SELECT * FROM comments WHERE status = 'active' + - sql: SELECT * FROM comments WHERE deleted_at IS NULL + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() + - sql: SELECT * FROM comments WHERE region = connection.parameter('region') + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() AND org_id = auth.parameter('org_id') + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() AND status = 'active' + - sql: SELECT * FROM comments WHERE list_id = subscription.parameter('list_id') AND list_id IN (SELECT id FROM lists WHERE owner_id = auth.user_id()) + - sql: SELECT * FROM comments WHERE owner_id = auth.user_id() OR shared_with = auth.user_id() + - sql: SELECT * FROM comments WHERE status = 'active' AND (owner_id = auth.user_id() OR shared_with = auth.user_id()) + - sql: SELECT * FROM comments WHERE status != 'archived' + - sql: SELECT * FROM comments WHERE deleted_at IS NOT NULL + - sql: SELECT * FROM comments WHERE category IN '["draft", "hidden"]' + - sql: SELECT * FROM comments WHERE category NOT IN '["draft", "hidden"]' rejected_syntax: # double quotes denote identifiers, not string literals @@ -44,6 +57,11 @@ query: - sql: SELECT * FROM comments WHERE issue_id IN (SELECT id FROM issues WHERE owner_id = auth.user_id() OR name = 'test') err: OR + # NOT IN with fixed value list + # parser: list not supported here + - sql: SELECT * FROM comments WHERE category NOT IN ('draft', 'hidden') + err: list not supported here + rejected_semantic: # parser: Negations are not allowed here - sql: SELECT * FROM comments WHERE issue_id NOT IN (SELECT id FROM issues WHERE owner_id = auth.user_id()) @@ -64,3 +82,8 @@ query: # parser: Function 'request.user_id' is not defined - sql: SELECT * FROM issues WHERE owner_id = request.user_id() err: not defined + + # NOT IN with parameter source + # parser: Negations are not allowed here + - sql: SELECT * FROM comments WHERE id NOT IN subscription.parameter('excluded_ids') + err: negations are not allowed diff --git a/where-clause-support.md b/where-clause-support.md new file mode 100644 index 000000000..088cb662f --- /dev/null +++ b/where-clause-support.md @@ -0,0 +1,167 @@ +### WHERE Clause Support + +Sync queries support a subset of standard SQL `WHERE` clause syntax. The allowed operators and combinations differ between Sync Streams and Sync Rules, and are more restrictive than standard SQL. + + + + +**`=` and `IS NULL`** + +Compare a row column against a static value, a parameter, or another column: + +```sql +-- Static value +WHERE + status = 'active' +WHERE + deleted_at IS NULL + -- Parameter +WHERE + owner_id = auth.user_id () +WHERE + region = connection.parameter ('region') +``` + +**`AND`** + +Fully supported. Each condition is independent — you can mix parameter comparisons, subqueries, and row-value conditions freely in the same clause. + +```sql +-- Two independent parameter conditions +WHERE + owner_id = auth.user_id () + AND org_id = auth.parameter ('org_id') + -- Parameter condition + row-value condition +WHERE + owner_id = auth.user_id () + AND status = 'active' + -- Parameter condition + subquery +WHERE + list_id = subscription.parameter ('list_id') + AND list_id IN ( + SELECT + id + FROM + lists + WHERE + owner_id = auth.user_id () + ) +``` + +**`OR`** + +Supported, including `OR` nested inside `AND`. PowerSync automatically rewrites combinations like `A AND (B OR C)` into separate branches before evaluating. + +```sql +-- Top-level OR +WHERE + owner_id = auth.user_id () + OR shared_with = auth.user_id () + -- OR nested inside AND +WHERE + status = 'active' + AND ( + owner_id = auth.user_id () + OR shared_with = auth.user_id () + ) +``` + +Each `OR` branch must be a valid filter on its own — you cannot have a branch that only makes sense in combination with the other branch. + +**`NOT`** + +Supported for simple conditions on row values: + +```sql +WHERE + status != 'archived' +WHERE + deleted_at IS NOT NULL +WHERE + category NOT IN ('draft', 'hidden') +``` + +Cannot negate an `IN` subquery or a parameter array expansion: + +```sql +-- Not supported +WHERE + issue_id NOT IN ( + SELECT + id + FROM + issues + WHERE + owner_id = auth.user_id () + ) + -- Not supported +WHERE + id NOT IN subscription.parameter ('excluded_ids') +``` + +`NOT IN` with a fixed value list works. `NOT IN` with a subquery or parameter does not. + + + + +**`=` and `IS NULL`** + +Compare a row column against a static value or a bucket parameter: + +```sql +-- Static value +WHERE + status = 'active' +WHERE + deleted_at IS NULL + -- Bucket parameter +WHERE + owner_id = bucket.user_id +``` + +**`AND`** + +Supported in both parameter queries and data queries. In parameter queries, each condition may match a different parameter. However, you cannot combine two `IN` expressions on parameters in the same `AND` — split them into separate parameter queries instead. + +```sql +-- Supported: parameter condition + row-value condition +WHERE + users.id = request.user_id () + AND users.is_admin = TRUE + -- Not supported: two IN expressions on parameters in the same AND +WHERE + bucket.list_id IN lists.allowed_ids + AND bucket.org_id IN lists.allowed_org_ids +``` + +**`OR`** + +Supported in parameter queries only when both sides of the `OR` reference the exact same set of parameters. In practice this is rarely useful — use separate parameter queries instead. + +```sql +-- Supported: both sides reference the same parameter +WHERE + lists.owner_id = request.user_id () + OR lists.shared_with = request.user_id () + -- Not supported: sides reference different parameters +WHERE + lists.owner_id = request.user_id () + OR lists.org_id = bucket.org_id +``` + +**`NOT`** + +Supported for simple row-value conditions in data queries. Not supported on parameter-matching expressions in parameter queries. + +```sql +-- Supported in data queries +WHERE + status != 'archived' +WHERE + deleted_at IS NOT NULL +WHERE + category NOT IN ('draft', 'hidden') + -- Not supported in parameter queries +WHERE + NOT users.is_admin = TRUE +``` From e6138436874a4d353cf013f96fdc39d271b8313b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 14:31:30 +0200 Subject: [PATCH 12/13] Fix for != operator. --- packages/sync-rules/src/compiler/sqlite.ts | 5 ++++- .../test/src/grammar_parity/fixtures/new_compiler.yaml | 7 ++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/sync-rules/src/compiler/sqlite.ts b/packages/sync-rules/src/compiler/sqlite.ts index d3d36f1b5..e7f604975 100644 --- a/packages/sync-rules/src/compiler/sqlite.ts +++ b/packages/sync-rules/src/compiler/sqlite.ts @@ -248,10 +248,13 @@ export class PostgresToSqlite { operand: { type: 'function', function: 'like', parameters: [left, right] } }; } else if (expr.op === '!=') { + const equals: SqlExpression = { type: 'binary', left, right, operator: '=' }; + this.options.locations.sourceForNode.set(equals, expr); + return { type: 'unary', operator: 'not', - operand: { type: 'binary', left, right, operator: '=' } + operand: equals }; } diff --git a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml index 95544b5ec..bb4e8c12a 100644 --- a/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml +++ b/packages/sync-rules/test/src/grammar_parity/fixtures/new_compiler.yaml @@ -37,12 +37,9 @@ query: - sql: SELECT * FROM comments WHERE status = 'active' AND (owner_id = auth.user_id() OR shared_with = auth.user_id()) - sql: SELECT * FROM comments WHERE deleted_at IS NOT NULL - sql: SELECT * FROM comments WHERE category IN '["draft", "hidden"]' - # FIXME: This one should work: - # - sql: SELECT * FROM comments WHERE status != 'archived' - # FIXME: This one should work: + - sql: SELECT * FROM comments WHERE status != 'archived' + # FIXME: This one should work # - sql: SELECT * FROM comments WHERE category NOT IN '["draft", "hidden"]' - # TODO: should we support this? - # - sql: SELECT id FROM assets WHERE category IN ('draft', 'hidden') rejected_syntax: # double quotes denote identifiers, not string literals From d0d945e2a9476ec616eda017aef34e5959bf178d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Feb 2026 14:39:31 +0200 Subject: [PATCH 13/13] Remove top-level declarations. --- packages/sync-rules/grammar/bucket-definitions.ebnf | 2 -- packages/sync-rules/grammar/sync-streams-alpha.ebnf | 2 -- packages/sync-rules/grammar/sync-streams-compiler.ebnf | 2 -- 3 files changed, 6 deletions(-) diff --git a/packages/sync-rules/grammar/bucket-definitions.ebnf b/packages/sync-rules/grammar/bucket-definitions.ebnf index a9b2b42ad..b92376ecc 100644 --- a/packages/sync-rules/grammar/bucket-definitions.ebnf +++ b/packages/sync-rules/grammar/bucket-definitions.ebnf @@ -7,8 +7,6 @@ - bucket_definitions..data[] */ -BucketDefinitionSql ::= ParameterQuery | DataQuery - /* Parameter-query SQL forms used under bucket_definitions..parameters */ ParameterQuery ::= TableValuedParameterQuery | TableParameterQuery | StaticParameterQuery diff --git a/packages/sync-rules/grammar/sync-streams-alpha.ebnf b/packages/sync-rules/grammar/sync-streams-alpha.ebnf index 296cd6eaf..5707ffada 100644 --- a/packages/sync-rules/grammar/sync-streams-alpha.ebnf +++ b/packages/sync-rules/grammar/sync-streams-alpha.ebnf @@ -9,8 +9,6 @@ - YAML keys `with` and `queries` are not supported in this mode. */ -SyncStreamsAlphaSql ::= SyncStreamsAlphaQuery - /* SQL form used under streams..query in sync streams alpha mode */ SyncStreamsAlphaQuery ::= "SELECT" StreamSelectList "FROM" TableRef ("WHERE" StreamWhereExpr)? diff --git a/packages/sync-rules/grammar/sync-streams-compiler.ebnf b/packages/sync-rules/grammar/sync-streams-compiler.ebnf index 5902af5ab..f1fdccfb2 100644 --- a/packages/sync-rules/grammar/sync-streams-compiler.ebnf +++ b/packages/sync-rules/grammar/sync-streams-compiler.ebnf @@ -10,8 +10,6 @@ - This grammar is used when config.sync_config_compiler = true (and edition >= 2) */ -SyncStreamsCompilerSql ::= CompilerStreamQuery | CompilerSubquery | CompilerCteSubquery - /* Top-level stream query form: streams..query / streams..queries[] */ CompilerStreamQuery ::= "SELECT" ResultColumnList "FROM" FromSource FromContinuation* ("WHERE" WhereExpr)?