Skip to content

parser: fix float lexing #532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kodiakhq merged 1 commit into from
Jun 7, 2025
Merged

parser: fix float lexing #532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions crates/squawk_lexer/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,14 @@ impl Cursor<'_> {
}
_ => (),
}
} else {
match self.first() {
'e' | 'E' => {
self.bump();
empty_exponent = !self.eat_float_exponent();
}
_ => (),
}
}
LiteralKind::Float {
base,
Expand Down Expand Up @@ -623,6 +631,7 @@ $foo$hello$world$bar$
1e-10
1e+10
1e10
4664.E+5
"#))
}

Expand Down
15 changes: 15 additions & 0 deletions crates/squawk_lexer/src/snapshots/squawk_lexer__tests__floats.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---
source: crates/squawk_lexer/src/lib.rs
expression: "lex(r#\"\n4664.E+5\n4664E+5\n4664E5\n4664e5\n \"#)"
---
[
"\n" @ Whitespace,
"4664.E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
"\n" @ Whitespace,
"4664E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
"\n" @ Whitespace,
"4664E5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
"\n" @ Whitespace,
"4664e5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
"\n " @ Whitespace,
]
4 changes: 3 additions & 1 deletion crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
source: crates/squawk_lexer/src/lib.rs
expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n1e10\n\"#)"
expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n1e10\n4664.E+5\n\"#)"
---
[
"\n" @ Whitespace,
Expand All @@ -24,4 +24,6 @@ expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n
"\n" @ Whitespace,
"1e10" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
"\n" @ Whitespace,
"4664.E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
"\n" @ Whitespace,
]
1 change: 1 addition & 0 deletions crates/squawk_parser/src/generated/syntax_kind.rs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

138 changes: 56 additions & 82 deletions crates/squawk_parser/src/grammar.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,19 @@ fn atom_expr(p: &mut Parser<'_>) -> Option<CompletedMarker> {
p.bump(POSITIONAL_PARAM);
m.complete(p, LITERAL)
}
(CAST_KW | TREAT_KW, L_PAREN) => {
let m = p.start();
p.bump_any();
p.bump(L_PAREN);
if expr(p).is_none() {
p.error("expected an expression");
}
p.expect(AS_KW);
type_name(p);
opt_collate(p);
p.expect(R_PAREN);
m.complete(p, CAST_EXPR)
}
(EXTRACT_KW, L_PAREN) => extract_fn(p),
(JSON_EXISTS_KW, L_PAREN) => json_exists_fn(p),
(JSON_ARRAY_KW, L_PAREN) => json_array_fn(p),
Expand Down Expand Up @@ -1260,20 +1273,6 @@ fn lhs(p: &mut Parser<'_>, r: &Restrictions) -> Option<CompletedMarker> {
p.bump_any();
(PREFIX_EXPR, 3)
}
CAST_KW | TREAT_KW => {
m = p.start();
p.bump_any();
p.expect(L_PAREN);
if expr(p).is_none() {
p.error("expected an expression");
}
p.expect(AS_KW);
type_name(p);
opt_collate(p);
p.expect(R_PAREN);
let cm = m.complete(p, CAST_EXPR);
return Some(cm);
}
OPERATOR_KW if p.at(OPERATOR_CALL) => {
m = p.start();
p.expect(OPERATOR_CALL);
Expand Down Expand Up @@ -1915,7 +1914,12 @@ fn call_expr_args(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
}
m.complete(p, OVER_CLAUSE);
}
m.complete(p, CALL_EXPR)
let cm = m.complete(p, CALL_EXPR);
if opt_string_literal(p).is_some() {
cm.precede(p).complete(p, CAST_EXPR)
} else {
cm
}
}

// foo[]
Expand Down Expand Up @@ -2017,13 +2021,13 @@ fn postfix_dot_expr<const FLOAT_RECOVERY: bool>(
if !FLOAT_RECOVERY {
assert!(p.at(DOT));
}
field_expr::<FLOAT_RECOVERY>(p, Some(lhs), allow_calls).map(|m| {
field_expr::<FLOAT_RECOVERY>(p, Some(lhs), allow_calls).map(|cm| {
// A field followed by a literal is a type cast so we insert a CAST_EXPR
// preceding it to wrap the previously parsed data.
if !p.at(NULL_KW) && !p.at(DEFAULT_KW) && literal(p).is_some() {
m.precede(p).complete(p, CAST_EXPR)
if opt_string_literal(p).is_some() {
cm.precede(p).complete(p, CAST_EXPR)
} else {
m
cm
}
})
}
Expand Down Expand Up @@ -4868,9 +4872,7 @@ fn savepoint(p: &mut Parser<'_>) -> CompletedMarker {
assert!(p.at(SAVEPOINT_KW));
let m = p.start();
p.bump(SAVEPOINT_KW);
if name_ref_(p).is_none() {
p.error("expected a name");
}
name_ref(p);
m.complete(p, SAVEPOINT)
}

Expand All @@ -4882,9 +4884,7 @@ fn release(p: &mut Parser<'_>) -> CompletedMarker {
let m = p.start();
p.bump(RELEASE_KW);
p.eat(SAVEPOINT_KW);
if name_ref_(p).is_none() {
p.error("expected a name");
}
name_ref(p);
m.complete(p, RELEASE_SAVEPOINT)
}

Expand All @@ -4909,9 +4909,7 @@ fn rollback(p: &mut Parser<'_>) -> CompletedMarker {
let _ = p.eat(WORK_KW) || p.eat(TRANSACTION_KW);
if is_rollback && p.eat(TO_KW) {
p.eat(SAVEPOINT_KW);
if name_ref_(p).is_none() {
p.error("expected a name");
}
name_ref(p);
} else if p.eat(AND_KW) {
p.eat(NO_KW);
p.expect(CHAIN_KW);
Expand Down Expand Up @@ -5275,15 +5273,7 @@ fn alter_server(p: &mut Parser<'_>) -> CompletedMarker {
string_literal(p);
found_option = true;
}
if p.eat(OPTIONS_KW) {
found_option = true;
p.expect(L_PAREN);
alter_option(p);
while !p.at(EOF) && p.eat(COMMA) {
alter_option(p);
}
p.expect(R_PAREN);
}
found_option |= opt_options_list(p);
if !found_option {
p.error("expected ALTER SERVER option");
}
Expand Down Expand Up @@ -7543,13 +7533,9 @@ fn alter_user_mapping(p: &mut Parser<'_>) -> CompletedMarker {
role(p);
p.expect(SERVER_KW);
name_ref(p);
p.expect(OPTIONS_KW);
p.expect(L_PAREN);
alter_option(p);
while !p.at(EOF) && p.eat(COMMA) {
alter_option(p);
if !opt_options_list(p) {
p.error("expected options");
}
p.expect(R_PAREN);
m.complete(p, ALTER_USER_MAPPING)
}

Expand Down Expand Up @@ -8337,13 +8323,9 @@ fn create_foreign_data_wrapper(p: &mut Parser<'_>) -> CompletedMarker {
fn opt_fdw_option(p: &mut Parser<'_>) -> bool {
match p.current() {
OPTIONS_KW => {
p.bump(OPTIONS_KW);
p.expect(L_PAREN);
alter_option(p);
while !p.at(EOF) && p.eat(COMMA) {
alter_option(p);
if !opt_options_list(p) {
p.error("expected options");
}
p.expect(R_PAREN);
true
}
HANDLER_KW | VALIDATOR_KW => {
Expand Down Expand Up @@ -8684,10 +8666,12 @@ fn publication_object(p: &mut Parser<'_>) {
if !p.eat(CURRENT_SCHEMA_KW) {
name_ref(p);
}
while !p.at(EOF) && p.eat(COMMA) {
if !p.eat(CURRENT_SCHEMA_KW) {
name_ref(p);
if p.eat(WHERE_KW) {
p.expect(L_PAREN);
if expr(p).is_none() {
p.error("expected expression");
}
p.expect(R_PAREN);
}
} else if p.eat(CURRENT_SCHEMA_KW) {
return;
Expand Down Expand Up @@ -9913,25 +9897,18 @@ fn explain_option(p: &mut Parser<'_>) {
}
}

// TODO: I think we want something like deliminated where we give it a FIRST
// token set so we can be robust to missing commas
fn one_or_more(p: &mut Parser<'_>, mut parse: impl FnMut(&mut Parser<'_>)) {
parse(p);
while !p.at(EOF) && p.eat(COMMA) {
parse(p);
}
}

// [ OPTIONS ( option 'value' [, ... ] ) ]
fn opt_options_list(p: &mut Parser<'_>) {
// [ OPTIONS ( option 'value' [, ... ] ) ]
fn opt_options_list(p: &mut Parser<'_>) -> bool {
if p.eat(OPTIONS_KW) {
p.expect(L_PAREN);
one_or_more(p, |p| {
col_label(p);
string_literal(p);
});
alter_option(p);
while !p.at(EOF) && p.eat(COMMA) {
alter_option(p);
}
p.expect(R_PAREN);
true
} else {
false
}
}

Expand Down Expand Up @@ -11900,15 +11877,9 @@ fn insert(p: &mut Parser<'_>, m: Option<Marker>) -> CompletedMarker {
// ( { index_column_name | ( index_expression ) } [ COLLATE collation ] [ opclass ] [, ...] ) [ WHERE index_predicate ]
} else if p.eat(L_PAREN) {
while !p.at(EOF) {
// ( index_expression )
if p.eat(L_PAREN) {
// TODO: more strict?
if expr(p).is_none() {
p.error("expected index_expression");
}
// index_column_name
if expr(p).is_none() {
p.error("expected expression");
}
name_ref(p);
opt_collate(p);
// [ opclass ]
p.eat(IDENT);
Expand Down Expand Up @@ -13358,6 +13329,7 @@ fn alter_table_action(p: &mut Parser<'_>) -> Option<SyntaxKind> {
// column_name
name_ref(p);
type_name(p);
opt_options_list(p);
opt_collate(p);
// [ column_constraint [ ... ] ]
while !p.at(EOF) {
Expand All @@ -13373,7 +13345,7 @@ fn alter_table_action(p: &mut Parser<'_>) -> Option<SyntaxKind> {
p.bump(ATTACH_KW);
p.expect(PARTITION_KW);
// name
name_ref(p);
path_name_ref(p);
// { FOR VALUES partition_bound_spec | DEFAULT }
partition_option(p);
ATTACH_PARTITION
Expand Down Expand Up @@ -13521,6 +13493,12 @@ fn alter_table_action(p: &mut Parser<'_>) -> Option<SyntaxKind> {
ALTER_COLUMN
}
}
OPTIONS_KW => {
if !opt_options_list(p) {
p.error("expected options list");
}
OPTIONS_LIST
}
_ => return None,
};
Some(kind)
Expand Down Expand Up @@ -13759,13 +13737,9 @@ fn alter_column_option(p: &mut Parser<'_>) -> Option<SyntaxKind> {
}
// OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ])
OPTIONS_KW => {
p.bump(OPTIONS_KW);
p.expect(L_PAREN);
alter_option(p);
while !p.at(EOF) && p.eat(COMMA) {
alter_option(p);
if !opt_options_list(p) {
p.error("expected options");
}
p.expect(R_PAREN);
SET_OPTIONS_LIST
}
// SET DEFAULT expression
Expand Down
27 changes: 26 additions & 1 deletion crates/squawk_parser/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,24 @@ impl<'t> Parser<'t> {
m.complete(self, SyntaxKind::IS_NOT);
return true;
}
// SyntaxKind::BYTE_STRING => {
// let m = self.start();
// self.bump(SyntaxKind::BYTE_STRING);
// if self.eat(SyntaxKind::UESCAPE_KW) {
// self.expect(SyntaxKind::STRING);
// }
// m.complete(self, SyntaxKind::BYTE_STRING);
// return true;
// }
// SyntaxKind::IDENT => {
// let m = self.start();
// self.bump(SyntaxKind::IDENT);
// if self.eat(SyntaxKind::UESCAPE_KW) {
// self.expect(SyntaxKind::STRING);
// }
// m.complete(self, SyntaxKind::IDENT);
// return true;
// }
SyntaxKind::CUSTOM_OP => {
let m = self.start();
while !self.at(SyntaxKind::EOF) {
Expand Down Expand Up @@ -412,7 +430,7 @@ impl<'t> Parser<'t> {
if kind == SyntaxKind::EOF {
return;
}
self.do_bump(kind, 1);
self.bump(kind);
}

/// Advances the parser by one token
Expand Down Expand Up @@ -568,6 +586,13 @@ impl<'t> Parser<'t> {
SyntaxKind::LIKE_KW,
TrivaBetween::Allowed,
),
// not ilike
SyntaxKind::NOT_ILIKE => self.at_composite2(
n,
SyntaxKind::NOT_KW,
SyntaxKind::ILIKE_KW,
TrivaBetween::Allowed,
),
// not in
SyntaxKind::NOT_IN => self.at_composite2(
n,
Expand Down
Loading
Loading