From 68142c373ad4552e57c7cea87bae0a5e74846b39 Mon Sep 17 00:00:00 2001 From: Nick Winans Date: Mon, 5 Jan 2026 20:59:41 -0800 Subject: [PATCH 1/3] Create lezer luau parser --- package-lock.json | 15 + package.json | 6 +- scripts/luau-corpus.luau | 67 +++++ scripts/test-luau-highlighting.mjs | 120 ++++++++ scripts/test-luau-parser.mjs | 11 + src/lib/codemirror-lang-luau/index.js | 78 +++++ src/lib/codemirror-lang-luau/luau.grammar | 283 +++++++++++++++++++ src/lib/codemirror-lang-luau/parser.js | 24 ++ src/lib/codemirror-lang-luau/parser.terms.js | 159 +++++++++++ src/lib/codemirror-lang-luau/tokenizer.js | 148 ++++++++++ 10 files changed, 910 insertions(+), 1 deletion(-) create mode 100644 scripts/luau-corpus.luau create mode 100644 scripts/test-luau-highlighting.mjs create mode 100644 scripts/test-luau-parser.mjs create mode 100644 src/lib/codemirror-lang-luau/index.js create mode 100644 src/lib/codemirror-lang-luau/luau.grammar create mode 100644 src/lib/codemirror-lang-luau/parser.js create mode 100644 src/lib/codemirror-lang-luau/parser.terms.js create mode 100644 src/lib/codemirror-lang-luau/tokenizer.js diff --git a/package-lock.json b/package-lock.json index 82c5f47..dd13146 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "vscode-textmate": "^9.3.0" }, "devDependencies": { + "@lezer/generator": "^1.8.0", "@sveltejs/vite-plugin-svelte": "^6.2.1", "@tailwindcss/vite": "^4.1.18", "@tsconfig/svelte": "^5.0.6", @@ -642,6 +643,20 @@ "integrity": "sha512-PNGcolp9hr4PJdXR4ix7XtixDrClScvtSCYW3rQG106oVMOOI+jFb+0+J3mbeL/53g1Zd6s0kJzaw6Ri68GmAA==", "license": "MIT" }, + "node_modules/@lezer/generator": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.8.0.tgz", + "integrity": "sha512-/SF4EDWowPqV1jOgoGSGTIFsE7Ezdr7ZYxyihl5eMKVO5tlnpIhFcDavgm1hHY5GEonoOAEnJ0CU0x+tvuAuUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.1.0", + "@lezer/lr": "^1.3.0" + }, + "bin": { + "lezer-generator": "src/lezer-generator.cjs" + } + }, "node_modules/@lezer/highlight": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.3.tgz", diff --git a/package.json b/package.json index d7d43e0..4474de0 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,13 @@ "prebuild": "npm run build:wasm", "build": "vite build", "preview": "vite preview", - "check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json" + "check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json", + "build:luau-parser": "lezer-generator src/lib/codemirror-lang-luau/luau.grammar -o src/lib/codemirror-lang-luau/parser.js", + "test:luau-parser": "node scripts/test-luau-parser.mjs", + "test:luau-highlighting": "node scripts/test-luau-highlighting.mjs" }, "devDependencies": { + "@lezer/generator": "^1.8.0", "@sveltejs/vite-plugin-svelte": "^6.2.1", "@tailwindcss/vite": "^4.1.18", "@tsconfig/svelte": "^5.0.6", diff --git a/scripts/luau-corpus.luau b/scripts/luau-corpus.luau new file mode 100644 index 0000000..45c8dee --- /dev/null +++ b/scripts/luau-corpus.luau @@ -0,0 +1,67 @@ +-- regular comment +--[=[ +long comment with [=[ nested text +]=] + +export type Result = { ok: boolean, value: T? } + +type Pair = { first: T, second: U } + +type function Mapper(value: T): U + +local function make(value: T): T + return value +end + +local function returnsNothing() + return +end + +local function packer(...: T...): T... + return ... +end + +local function returnsTuple(): (number, string, ...boolean) + return 1, "ok", true +end + +-- highlighting corpus +local number = 10 +local type = "identifier" +local continue = 42 + +local function makeFoo(bar: number, ...: T): T + return bar +end + +local made = makeFoo(1) + +type Vec2 = { read x: number, write y: string } +export type Alias = Vec2 + +type Callback = (x: number, y: string) -> number + +local typed: typeof(number) = number +local casted = number :: Vec2 + +number += 5 +number //= 2 +number ..= "!" + +local msg = `value is {number}, next is {number + 1}` +local empty = `` + +local long = [=[line one +line two]=] + +for i = 1, 10 do + if i % 2 == 0 then + continue + end +end + +for _, n in numbers do + sum += n +end + +local t = { a = 1, [2] = 3 } diff --git a/scripts/test-luau-highlighting.mjs b/scripts/test-luau-highlighting.mjs new file mode 100644 index 0000000..13a2b86 --- /dev/null +++ b/scripts/test-luau-highlighting.mjs @@ -0,0 +1,120 @@ +import { readFileSync } from "node:fs"; +import assert from "node:assert/strict"; +import { highlightTree, tagHighlighter, tags as t } from "@lezer/highlight"; +import { luauLanguage } from "../src/lib/codemirror-lang-luau/index.js"; + +const source = readFileSync(new URL("./luau-corpus.luau", import.meta.url), "utf8"); +const tree = luauLanguage.parser.parse(source); + +const testHighlighter = tagHighlighter([ + { tag: t.function(t.variableName), class: "fn" }, + { tag: t.definition(t.variableName), class: "var-def" }, + { tag: t.definition(t.typeName), class: "type-def" }, + { tag: t.variableName, class: "var" }, + { tag: t.typeName, class: "type" }, + { tag: t.controlKeyword, class: "kw" }, + { tag: t.definitionKeyword, class: "kw" }, + { tag: t.operatorKeyword, class: "kw" }, + { tag: t.keyword, class: "kw" }, + { tag: t.modifier, class: "modifier" }, + { tag: t.operator, class: "op" }, + { tag: t.definitionOperator, class: "op" }, + { tag: t.compareOperator, class: "op" }, + { tag: t.arithmeticOperator, class: "op" }, + { tag: t.bitwiseOperator, class: "op" }, + { tag: t.typeOperator, class: "op" }, + { tag: t.string, class: "string" }, + { tag: t.special(t.string), class: "string" }, + { tag: t.number, class: "number" }, + { tag: t.comment, class: "comment" }, + { tag: t.punctuation, class: "punct" }, + { tag: t.meta, class: "meta" }, +]); + +const classes = Array.from({ length: source.length }, () => new Set()); + +highlightTree(tree, testHighlighter, (from, to, cls) => { + const names = cls.split(" ").filter(Boolean); + for (let i = from; i < to; i++) { + for (const name of names) { + classes[i].add(name); + } + } +}); + +function findAnchor(anchor) { + const start = source.indexOf(anchor); + assert.ok(start >= 0, `anchor not found: ${anchor}`); + return start; +} + +function assertTokenClass({ anchor, token, className }) { + const start = findAnchor(anchor); + const tokenOffset = anchor.indexOf(token); + let from; + if (tokenOffset >= 0) { + from = start + tokenOffset; + } else { + const found = source.indexOf(token, start); + assert.ok(found >= 0, `token not found after anchor: ${token} in ${anchor}`); + from = found; + } + const to = from + token.length; + for (let i = from; i < to; i++) { + assert.ok( + classes[i].has(className), + `expected ${className} on '${token}' at ${from}, got ${Array.from(classes[i]).join(", ")}` + ); + } +} + +const expectations = [ + { anchor: "local number = 10", token: "local", className: "kw" }, + { anchor: "local number = 10", token: "number", className: "var" }, + { anchor: "local number = 10", token: "=", className: "op" }, + { anchor: "local type = \"identifier\"", token: "type", className: "var" }, + { anchor: "local type = \"identifier\"", token: "identifier", className: "string" }, + { anchor: "local continue = 42", token: "continue", className: "var" }, + { anchor: "\n continue\n", token: "continue", className: "kw" }, + { anchor: "local function makeFoo", token: "function", className: "kw" }, + { anchor: "local function makeFoo", token: "makeFoo", className: "fn" }, + { anchor: "makeFoo(bar: number, ...: T): T", token: "T", className: "type-def" }, + { anchor: "makeFoo(bar: number, ...: T): T", token: "bar", className: "var-def" }, + { anchor: "makeFoo(bar: number, ...: T): T", token: "number", className: "type" }, + { anchor: "...: T): T", token: "T", className: "type" }, + { anchor: "local made = makeFoo(1)", token: "makeFoo", className: "fn" }, + { anchor: "type Vec2 =", token: "type", className: "kw" }, + { anchor: "type Vec2 =", token: "Vec2", className: "type-def" }, + { anchor: "export type Alias", token: "export", className: "kw" }, + { anchor: "export type Alias", token: "type", className: "kw" }, + { anchor: "export type Alias", token: "Alias", className: "type-def" }, + { anchor: "export type Alias", token: "Vec2", className: "type" }, + { anchor: "packer(...: T...): T...", token: "T", className: "type-def" }, + { anchor: "...: T...): T...", token: "T", className: "type" }, + { anchor: "type Callback = (x: number, y: string) -> number", token: "x", className: "var-def" }, + { anchor: "number, y: string", token: "y", className: "var-def" }, + { anchor: "type Callback = (x: number, y: string) -> number", token: "->", className: "op" }, + { anchor: "returnsTuple(): (number, string, ...boolean)", token: "number", className: "type" }, + { anchor: "read x: number", token: "read", className: "modifier" }, + { anchor: "write y: string", token: "write", className: "modifier" }, + { anchor: "local typed: typeof(number)", token: "typeof", className: "kw" }, + { anchor: "local typed: typeof(number)", token: "number", className: "var" }, + { anchor: "local casted = number :: Vec2", token: "::", className: "op" }, + { anchor: "local casted = number :: Vec2", token: "Vec2", className: "type" }, + { anchor: "number += 5", token: "+=", className: "op" }, + { anchor: "number //= 2", token: "//=", className: "op" }, + { anchor: "number ..= \"!\"", token: "..=", className: "op" }, + { anchor: "\n return\nend\n\nlocal function packer", token: "return", className: "kw" }, + { anchor: "local msg = `value is", token: "`value is ", className: "string" }, + { anchor: "local long = [=[line one", token: "[=[line one\nline two]=]", className: "string" }, + { anchor: "-- regular comment", token: "-- regular comment", className: "comment" }, + { anchor: "--[=[\nlong comment", token: "--[=[\nlong comment with [=[ nested text\n]=]", className: "comment" }, + { anchor: "for _, n in numbers do", token: "_", className: "var-def" }, + { anchor: "for _, n in numbers do", token: "n", className: "var-def" }, +]; + +for (const expectation of expectations) { + assertTokenClass(expectation); +} + +console.log(`Luau highlight test passed (${expectations.length} checks).`); diff --git a/scripts/test-luau-parser.mjs b/scripts/test-luau-parser.mjs new file mode 100644 index 0000000..7553734 --- /dev/null +++ b/scripts/test-luau-parser.mjs @@ -0,0 +1,11 @@ +import { readFileSync } from "node:fs"; +import assert from "node:assert/strict"; +import { parser } from "../src/lib/codemirror-lang-luau/parser.js"; + +const source = readFileSync(new URL("./luau-corpus.luau", import.meta.url), "utf8"); +const tree = parser.parse(source); + +assert.ok(tree, "parser returned a tree"); +assert.equal(tree.topNode.type.name, "Chunk", "root node is Chunk"); + +console.log(`Parsed Luau corpus: ${tree.topNode.type.name} (${tree.length} chars)`); diff --git a/src/lib/codemirror-lang-luau/index.js b/src/lib/codemirror-lang-luau/index.js new file mode 100644 index 0000000..9a85f17 --- /dev/null +++ b/src/lib/codemirror-lang-luau/index.js @@ -0,0 +1,78 @@ +import { LRLanguage, LanguageSupport } from "@codemirror/language"; +import { styleTags, tags as t } from "@lezer/highlight"; +import { parser } from "./parser.js"; + +const luauHighlight = styleTags({ + "do then end while for in repeat until if elseif else return break": t.controlKeyword, + "local function": t.definitionKeyword, + "and or not": t.operatorKeyword, + "read write": t.modifier, + "typeof": t.operatorKeyword, + "ContinueStatement/continue": t.controlKeyword, + "TypeDeclaration/type ExportTypeDeclaration/type TypeFunctionDeclaration/type ExportTypeFunctionDeclaration/type": t.definitionKeyword, + "ExportTypeDeclaration/export ExportTypeFunctionDeclaration/export": t.definitionKeyword, + "true false": t.bool, + "nil": t.null, + Number: t.number, + String: t.string, + StringContentDouble: t.string, + StringContentSingle: t.string, + Escape: t.string, + LongString: t.string, + InterpChunk: t.string, + InterpStart: t.string, + InterpEnd: t.string, + LineComment: t.lineComment, + LongComment: t.blockComment, + Identifier: t.variableName, + Name: t.variableName, + "Param/Binding/Name/Identifier": t.definition(t.variableName), + "ForNumericStatement/Binding/Name/Identifier": t.definition(t.variableName), + "ForGenericStatement/BindingList/Binding/Name/Identifier": t.definition(t.variableName), + TypeName: t.typeName, + "TypeName/Identifier TypeName/Name/Identifier": t.typeName, + "GenericTypePack/Name/Identifier": t.typeName, + "TypeDeclaration/Identifier ExportTypeDeclaration/Identifier TypeFunctionDeclaration/Identifier ExportTypeFunctionDeclaration/Identifier": t.definition(t.typeName), + "TypeDeclaration/Name/Identifier ExportTypeDeclaration/Name/Identifier TypeFunctionDeclaration/Name/Identifier ExportTypeFunctionDeclaration/Name/Identifier": t.definition(t.typeName), + "GenericTypeParam/Identifier GenericTypeParam/Name/Identifier": t.definition(t.typeName), + "GenericTypeParamWithDefault/Identifier GenericTypeParamWithDefault/Name/Identifier": t.definition(t.typeName), + "GenericTypeParam/GenericTypePack/Name/Identifier GenericTypeParamWithDefault/GenericTypePack/Name/Identifier": t.definition(t.typeName), + "FunctionName/Identifier FunctionName/Name/Identifier": t.function(t.variableName), + "LocalFunctionDeclaration/Identifier LocalFunctionDeclaration/Name/Identifier": t.function(t.variableName), + "CallExpression/PrimaryExpression/Name CallExpression/PrimaryExpression/Name/Identifier": t.function(t.variableName), + "CallExpression/FieldSuffix/Name CallExpression/FieldSuffix/Name/Identifier": t.function(t.variableName), + "CallSuffix/Name CallSuffix/Name/Identifier": t.function(t.variableName), + "BoundType/Identifier BoundType/Name/Identifier": t.definition(t.variableName), + Attribute: t.meta, + AssignOp: t.definitionOperator, + CompoundAssignOp: t.definitionOperator, + CompareOp: t.compareOperator, + ShiftOp: t.bitwiseOperator, + UnaryOp: t.operator, + "TypeAnnotation/TypeColon ReturnType/TypeColon VarArgAnnotation/TypeColon TableProp/TypeColon TableIndexer/TypeColon BoundType/TypeColon": t.typeOperator, + "TypeCast TypeArrow": t.typeOperator, + "..": t.arithmeticOperator, + ".": t.derefOperator, + ", ;": t.separator, + TypeColon: t.punctuation, + ParenL: t.paren, + ParenR: t.paren, + BracketL: t.squareBracket, + BracketR: t.squareBracket, + BraceL: t.brace, + BraceR: t.brace, + "InterpOpen InterpClose": t.special(t.brace), +}); + +export const luauLanguage = LRLanguage.define({ + parser: parser.configure({ + props: [luauHighlight], + }), + languageData: { + commentTokens: { line: "--", block: { open: "--[[", close: "]]" } }, + }, +}); + +export function luau() { + return new LanguageSupport(luauLanguage); +} diff --git a/src/lib/codemirror-lang-luau/luau.grammar b/src/lib/codemirror-lang-luau/luau.grammar new file mode 100644 index 0000000..e5d8aca --- /dev/null +++ b/src/lib/codemirror-lang-luau/luau.grammar @@ -0,0 +1,283 @@ +@precedence { + or @left, + and @left, + compare @left, + shift @left, + concat @right, + plus @left, + times @left, + power @right, + call, + prefix, + as @left, + optional @right, + typeAnd @left, + typeOr @left, + typeargs +} + +@top Chunk { Block } + +Block { Statement* LastStatement? } + +Statement { + SimpleStatement | + DoStatement | + WhileStatement | + RepeatStatement | + IfStatement | + ForNumericStatement | + ForGenericStatement | + FunctionDeclaration | + LocalFunctionDeclaration | + LocalDeclaration | + TypeDeclaration | + ExportTypeDeclaration | + TypeFunctionDeclaration | + ExportTypeFunctionDeclaration +} + +LastStatement { + ReturnStatement | + BreakStatement | + ContinueStatement +} + +SimpleStatement { + VarList AssignOp ExpList | + AssignableExpression CompoundAssignOp Expression | + CallStatement +} + +ReturnStatement { kw<"return"> ExpList? } +BreakStatement { kw<"break"> } +ContinueStatement { ckw<"continue"> } + +DoStatement { kw<"do"> Block kw<"end"> } +WhileStatement { kw<"while"> Expression kw<"do"> Block kw<"end"> } +RepeatStatement { kw<"repeat"> Block kw<"until"> Expression } +IfStatement { kw<"if"> Expression kw<"then"> Block (kw<"elseif"> Expression kw<"then"> Block)* (kw<"else"> Block)? kw<"end"> } +ForNumericStatement { kw<"for"> Binding AssignOp Expression "," Expression ("," Expression)? kw<"do"> Block kw<"end"> } +ForGenericStatement { kw<"for"> BindingList kw<"in"> ExpList kw<"do"> Block kw<"end"> } + +FunctionDeclaration { Attributes? kw<"function"> FunctionName FunctionBody } +LocalFunctionDeclaration { Attributes? kw<"local"> kw<"function"> Name FunctionBody } + +LocalDeclaration { kw<"local"> BindingList (AssignOp ExpList)? } + +TypeDeclaration { ckw<"type"> Name TypeParamsWithDefaults? AssignOp Type } +ExportTypeDeclaration { ckw<"export"> ckw<"type"> Name TypeParamsWithDefaults? AssignOp Type } +TypeFunctionDeclaration { ckw<"type"> kw<"function"> Name FunctionBody } +ExportTypeFunctionDeclaration { ckw<"export"> ckw<"type"> kw<"function"> Name FunctionBody } + +FunctionName { Name ("." Name)* (TypeColon Name)? } + +FunctionBody { TypeParams? ParamList ReturnType? Block kw<"end"> } + +ParamList { ParenL ParamListInner? ParenR } +ParamListInner { Param ("," Param)* } +Param { Binding | VarArgParam } +VarArgParam { "..." VarArgAnnotation? } +VarArgAnnotation { TypeColon VarArgType } +VarArgType { GenericTypePack | Type } + +BindingList { Binding ("," Binding)* } +Binding { Name TypeAnnotation? } +TypeAnnotation { TypeColon Type } + +ReturnType { TypeColon ReturnTypeValue } +ReturnTypeValue { TypeOrPack } + +VarList { AssignableExpression ("," AssignableExpression)* } +AssignableExpression { PrimaryExpression FieldSuffix* } +FieldExpression { PrimaryExpression !call FieldSuffix+ } + +CallStatement { CallExpression } + +FieldSuffix { "." Name | BracketL Expression BracketR } +CallSuffix { FunctionArgs | TypeColon Name FunctionArgs } +CallExpression { + PrimaryExpression !call CallSuffix FieldSuffix* | + PrimaryExpression !call FieldSuffix+ CallSuffix FieldSuffix* +} + +ParenthesizedExpression { ParenL Expression ParenR } +PrimaryExpression { Name | ParenthesizedExpression } + +FunctionArgs { ParenL ExpList? ParenR | TableConstructor | String | LongString | InterpolatedString } + +Expression { BinaryExpression | UnaryExpression | AsExpression | SimpleExpression } + +BinaryExpression { + Expression !or kw<"or"> Expression | + Expression !and kw<"and"> Expression | + Expression !compare CompareOp Expression | + Expression !shift ShiftOp Expression | + Expression !concat ".." Expression | + Expression !plus ("+" | "-") Expression | + Expression !times ("*" | "/" | "//" | "%") Expression | + Expression !power "^" Expression +} + +UnaryExpression { !prefix UnaryOp Expression } +UnaryOp { "-" | kw<"not"> | "#" | "~" } + +AsExpression { SimpleExpression !as TypeCast Type } + +CompareOp { "==" | "~=" | "<" | "<=" | ">" | ">=" } +ShiftOp { "<<" | ">>" | ">>>" } + +SimpleExpression { + Number | + String | + LongString | + InterpolatedString | + kw<"nil"> | + kw<"true"> | + kw<"false"> | + "..." | + TableConstructor | + FunctionExpression | + PrimaryExpression | + FieldExpression | + CallExpression | + IfExpression +} + +FunctionExpression { Attributes? kw<"function"> FunctionBody } + +IfExpression { kw<"if"> Expression kw<"then"> Expression (kw<"elseif"> Expression kw<"then"> Expression)* kw<"else"> Expression } + +ExpList { Expression ("," Expression)* } + +TableConstructor { BraceL FieldList? BraceR } +FieldList { Field (FieldSep Field)* FieldSep? } +Field { BracketL Expression BracketR AssignOp Expression | Name AssignOp Expression | Expression } +FieldSep { "," | ";" } + +Attributes { Attribute+ } +Attribute { "@" Name | "@[" AttributeParamList? "]" } +AttributeParamList { AttributeParam ("," AttributeParam)* } +AttributeParam { Name AttributeArgs? } +AttributeArgs { ParenL LiteralList? ParenR | TableConstructor | String | LongString } + +LiteralList { Literal ("," Literal)* } +Literal { kw<"nil"> | kw<"true"> | kw<"false"> | Number | String | LongString | TableConstructor } + +InterpolatedString[isolate] { InterpStart InterpPart* InterpEnd } +InterpPart { InterpChunk | Interpolation } +Interpolation { InterpOpen Expression InterpClose } + +Type { UnionType } +UnionType { UnionType !typeOr "|" IntersectionType | IntersectionType } +IntersectionType { IntersectionType !typeAnd "&" OptionalType | OptionalType } +OptionalType { SimpleType | SimpleType !optional "?" } + +SimpleType { + kw<"nil"> | + SingletonType | + kw<"typeof"> ParenL Expression ParenR | + TypeReference | + TableType | + FunctionType | + ParenthesizedType +} + +SingletonType { String | kw<"true"> | kw<"false"> } + +TypeReference { TypeName | TypeName !typeargs TypeArgs } +TypeName { Name ("." Name)* } +TypeArgs { "<" TypeParamList? ">" } +TypeParamList { TypeParam ("," TypeParam)* } +TypeParam { Type | TypePack | VariadicTypePack | GenericTypePack } + +ParenthesizedType { ParenL Type ParenR } + +TypePack { ParenL TypePackList? ParenR } +TypePackList { Type ("," (Type | VariadicTypePack))+ } + +VariadicTypePack { "..." Type } +GenericTypePack { Name "..." } + +TableType { BraceL Type BraceR | BraceL PropList? BraceR } +PropList { TableProp (FieldSep TableProp)* FieldSep? | TableIndexer (FieldSep TableProp)* FieldSep? } +TableProp { ReadWrite? Name TypeColon Type } +TableIndexer { ReadWrite? BracketL Type BracketR TypeColon Type } +ReadWrite { ckw<"read"> | ckw<"write"> } + +BoundTypeList { BoundType ("," BoundType)* | GenericTypePack | VariadicTypePack } +BoundType { Name TypeColon Type } + +FunctionType { TypeParams? ParenL BoundTypeList? ParenR TypeArrow ReturnTypeValue } + +TypeParams { "<" GenericTypeList? ">" } +GenericTypeList { GenericTypeParam ("," GenericTypeParam)* } +GenericTypeParam { Name | GenericTypePack } + +TypeParamsWithDefaults { "<" GenericTypeListWithDefaults? ">" } +GenericTypeListWithDefaults { GenericTypeParamWithDefault ("," GenericTypeParamWithDefault)* } +GenericTypeParamWithDefault { Name (AssignOp Type)? | GenericTypePack AssignOp TypePackDefault } +TypePackDefault { TypePack | VariadicTypePack | GenericTypePack } +TypeOrPack { Type | TypePack | VariadicTypePack | GenericTypePack } + +kw { @specialize[@name={term}] } +ckw { @extend[@name={term}] } + +Name { Identifier } + +AssignOp { "=" } +CompoundAssignOp { "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "^=" | "..=" } + +@external tokens longBracket from "./tokenizer.js" { LongString, LongComment } +@external tokens backtickTokens from "./tokenizer.js" { InterpStart, InterpChunk, InterpEnd, InterpOpen, InterpClose } +@context luauContext from "./tokenizer.js" + +@skip { Space | Newline | LineComment | LongComment } + +@tokens { + Space { $[\u0009\u000b\u000c\u0020\u00a0]+ } + Newline { $[\r\n] } + + LineComment { "--" ![\n]* } + + Number { + "0x" $[0-9a-fA-F]+ ("." $[0-9a-fA-F]+)? (("p" | "P") ("+" | "-")? @digit+)? | + @digit+ ("." @digit*)? (("e" | "E") ("+" | "-")? @digit+)? | + "." @digit+ (("e" | "E") ("+" | "-")? @digit+)? + } + + identifierStart { @asciiLetter | "_" } + identifierChar { identifierStart | @digit } + Identifier { identifierStart identifierChar* } + + Escape { "\\" ![\n] } + StringContentDouble { ![\\\n"]+ } + StringContentSingle { ![\\\n']+ } + + BraceL { "{" } + BraceR { "}" } + ParenL { "(" } + ParenR { ")" } + BracketL { "[" } + BracketR { "]" } + + TypeArrow { "->" } + TypeCast { "::" } + TypeColon { ":" } + + @precedence { "...", "..", "." } + @precedence { "//=", "//", "/" } + @precedence { "<<", "<=", "<" } + @precedence { ">>>", ">>", ">=", ">" } + @precedence { TypeArrow, "+=", "+", "-=", "-" } + @precedence { "..=", ".." } + @precedence { TypeCast, TypeColon } + @precedence { LineComment, "-" } +} + +@skip {} { + String[isolate] { + '"' (StringContentDouble | Escape)* ('"' | "\n") | + "'" (StringContentSingle | Escape)* ("'" | "\n") + } +} diff --git a/src/lib/codemirror-lang-luau/parser.js b/src/lib/codemirror-lang-luau/parser.js new file mode 100644 index 0000000..155bd3d --- /dev/null +++ b/src/lib/codemirror-lang-luau/parser.js @@ -0,0 +1,24 @@ +// @ts-nocheck +// This file was generated by lezer-generator. You probably shouldn't edit it. +import {LRParser} from "@lezer/lr" +import {longBracket, backtickTokens, luauContext} from "./tokenizer.js" +const spec_Identifier = {__proto__:null,or:48, and:50, not:60, nil:82, true:84, false:86, function:124, typeof:160, read:187, write:189, end:214, if:230, then:232, elseif:234, else:236, do:246, while:250, repeat:254, until:256, for:262, in:268, local:276, type:283, export:295, return:304, break:308, continue:313} +export const parser = LRParser.deserialize({ + version: 14, + states: "!6pO!dQTOOOOQR'#Cn'#CnO!kQTO'#DeO!pQTO'#DeOOQP'#Ft'#FtO!xQTO'#DdO#uQVO'#ClO$iQZO'#CpOOQR'#Cm'#CmO%yQTO'#CkOOQP'#Ey'#EyO&kQTO'#CjOOQP'#Cj'#CjOOQP'#Ci'#CiOOQP'#Fo'#FoO&pQTO'#ChOOQP'#Fh'#FhOOQP'#Ch'#ChQOQTOOO'TQTO'#EzO$iQZO'#E|O'[QTO'#FOO$iQZO'#FRO!kQTO'#FUO!kQTO'#FXO'cQTO'#FXO'kQTO'#F]O'sQTO'#F^O'{QTO'#FdO(QQZO'#FiOOQP'#Fk'#FkOOQP'#Fm'#FmOOQP,5:P,5:PO(eQTO'#DgO(|QTO'#DfO)UQTO,5:POOQP-E9r-E9rO)ZO`O'#DOO)iOpO'#DOO)wQVO'#DSO*SQZO'#DYO!kQTO'#EmO$iQZO'#EmOOQR'#GO'#GOO*^QVO,59WO+QQZO'#EpOOQR'#Ep'#EpOOQR'#Eo'#EoO!kQTO'#EoO+XQTO,5;YO.PQ!cO,59[OOQR'#Cy'#CyO$iQZO'#CxO.WQ!eO'#C|OOQR'#C|'#C|O0cQ!eO'#CrOOQR'#Cr'#CrO3kQTO'#DcO3sQTO'#DcO$iQZO'#ErO3xQTO'#GRO4QQTO,59VOOQR'#Ex'#ExO4YQZO,59UOOQR'#D`'#D`O4YQZO,59UOOQP-E9m-E9mOOQP,59S,59SO5jQTO,5;fO5oQ!cO,5;hO5vQTO,5;jO5{Q!cO,5;mO6SQTO'#DuO7vQTO,5;nO8RQTO,5;pO8WQTO'#FYO3kQTO,5;sO!kQTO,5;sO8fQTO,5;uO8kQTO'#FVO!kQTO,5;uO:UQTO,5;wO;lQTO,5;xO!kQTO,5RQVO,59nOOQR,59n,59nO>^Q!cO'#D]O$iQZO'#D]O>kQ!eO'#CmO@tQTO'#D[OOQR,59t,59tOAPQTO,59tOOQR,5;X,5;XOAUQ!cO,5;XOOQR-E9|-E9|OA]QTO1G0tOOQR,5;[,5;[OBvQTO,5;[OB{QVO,5;ZOA]QTO1G0tOOQR'#Cv'#CvOOQR'#Cw'#CwO$iQZO,59_O$iQZO,59_O$iQZO,59_O$iQZO,59_O$iQZO,59_OOQR1G.v1G.vO$iQZO,59_O$iQZO,59_OCaQ!eO,59dODaQ!eO,5;WOFlQ!eO,5;YOHeQTO,59gOIVQTO'#DnOI_QTO'#DrOIjQTO'#DmOItQTO'#DmOOQR,59},59}O3kQTO,59}OIyQ!cO,5;^OJQQTO'#ClOOQP,5VQ!eO'#C|O4YQZO,59_O4YQZO,59_O4YQZO,59_O4YQZO,59_O4YQZO,59_O4YQZO,59_O4YQZO,59_O#@zQ!cO,59dO4YQZO'#GPO#BgQTO,5;]O#F`Q!cO1G.yO#IPQ!cO1G.yO#IaQ!cO1G.yO#LWQ!cO1G.yO#LkQ!cO1G.yO#MOQ!cO1G.yO#MVQ!cO,5_P:g>k@VApAsAyA|BPPPBVBoDODRDXESEaEdEjEwPEzGsHrIzJ}LSPLSMTNYN_NbNhN{! RLS! t! w!!PPP!!XLS!![!!qP!#P!#cLS!#o!#s!#v!#yP:g!$P!$c!%}!&]!&h:gPPPPP!&w!&z&ZP&ZP&ZPP&Z&ZP&Z!'^P&Z!'d&ZP&Z&ZP!'j!'p!'s!'y&ZP&Z&Z!'|!(bP!(bP!(bP!(t!)Y!)`!)f!)l!)r!+_!+e!+k!+q!,T!,[!,b!,h!,n!,t!-h!-r!-|!.S!.Y!.aQbOQ!fcQ!heQ%x#yQ&O$UQ&Q$WS's%y%zQ)U'tQ)^'zQ)a'}Q*X)[Q*p*YQ*q*[R*{*xq^O_ce#y$U$W%y%z't'z'})[*Y*[*xq]O_ce#y$U$W%y%z't'z'})[*Y*[*xqZO_ce#y$U$W%y%z't'z'})[*Y*[*xpXO_ce#y$U$W%y%z't'z'})[*Y*[*xR$P!^pUO_ce#y$U$W%y%z't'z'})[*Y*[*x!l!VVdfmxz}!U!]#U#[#k#l#m#n#o#q#r$]$^$m$|%O%|&y'c'x'|(](^)W)X*U*]*n,Y,[Q$O!^o+^!a!c$V$f+_+`+a+b+c+d+e+g+t+w+z#vWOV_cdefmz}!U!]!^!a!c#U#[#k#l#m#n#o#q#r#y$U$V$W$]$^$f$m$|%y%z%|'c't'x'z'|'}(])W)X)[*U*Y*[*]*n*x+_+`+a+b+c+d+e+g+t+w+z,Y,[QpQSqR!|Y!jgj#x$Z'jS!mh!oQ!tkW#]x%O&y(^Q#ayQ#g!QQ$e!rQ$j!uQ$k!vb%a#v'R'a'b(e+O+P+R+TS%m#w'ep&R$X$h&k&m'X(Q(X(c(n)d)n*P*e*s+Q+SQ&]$_Q&`$aQ&b$dS&e$g(SQ&o$lQ'P%c['W%d'n+U+V+|+}^'m%w&}'h)j+W+Y+[Q(O&_S(d'Q)rU(u'_,Q,RQ)c(RU)q(k)t*gQ)w(p])z(r)|*j+X+Z+]$PWOV_cdefmxz}!U!]!^!a!c#U#[#k#l#m#n#o#q#r#y$U$V$W$]$^$f$m$|%O%y%z%|&y'c't'x'z'|'}(](^)W)X)[*U*Y*[*]*n*x+_+`+a+b+c+d+e+g+t+w+z,Y,[Q!SVQ!gdQ!ifU!wm}$^W#Zx%O&y(^Q#bzQ#s!UQ#}!]Q$R!aQ$x#UQ${#[Q%V#kQ%W#lQ%X#mQ%Y#nQ%Z#oS%[#q+dQ%]#rQ&P$VQ&Z$]Q&p$mQ&w$|Q'u%|Q(y'cQ)]'xQ)`'|Q)i(]Q*V)WQ*W)XQ*m*UQ*r*]Q*v*nQ+f+zQ+i+_Q+j+`Q+k+aQ+l+bQ+m+cQ+n+eQ+o+gQ+x+tQ+y+wS+{!c$fQ,S,YR,Z,[#]!YVdfmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r$V$]$^$f$m$|%O%|&y'c'x'|(](^)W)X*U*]*n+_+`+a+b+c+d+e+g+t+w+z,Y,[![#k!S!g!i!w#Z#b#s#}$x${%W%X%Y%Z&Z&p&w'u(y)])`)i*V*W*m*r*v,S,Zg+_$R&P+f+j+k+l+m+o+x+y+{!Y#l!S!g!i!w#Z#b#s#}$x${%X%Y%Z&Z&p&w'u(y)])`)i*V*W*m*r*v,S,Ze+`$R&P+f+k+l+m+o+x+y+{!l!UVdfmxz}!U!]#U#[#k#l#m#n#o#q#r$]$^$m$|%O%|&y'c'x'|(](^)W)X*U*]*n,Y,[o+z!a!c$V$f+_+`+a+b+c+d+e+g+t+w+z#]!XVdfmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r$V$]$^$f$m$|%O%|&y'c'x'|(](^)W)X*U*]*n+_+`+a+b+c+d+e+g+t+w+z,Y,[^!OU|!V#g#t*}+^#[!WVdfmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r$V$]$^$f$m$|%O%|&y'c'x'|(](^)W)X*U*]*n+_+`+a+b+c+d+e+g+t+w+z,Y,[Q!zqS$o!y&r!}%`#v$X$h%c%d%w&k&m&}'R'X'a'b'h'n(Q(X(c(e(n(r)d)j)n)|*P*e*j*s+O+P+Q+R+S+T+U+V+W+X+Y+Z+[+]+|+}^!OU|!V#g#t*}+^#]!WVdfmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r$V$]$^$f$m$|%O%|&y'c'x'|(](^)W)X*U*]*n+_+`+a+b+c+d+e+g+t+w+z,Y,[T#Ww#XT#Vw#X^!OU|!V#g#t*}+^#[!WVdfmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r$V$]$^$f$m$|%O%|&y'c'x'|(](^)W)X*U*]*n+_+`+a+b+c+d+e+g+t+w+z,Y,[Q!zqT$o!y&rR#`xQ#^xV&x%O&y(^Q!cZQ$]!kQ$f!sQ$h!tQ$|#]Q&k$iQ&m$kQ(Q&eQ(R&fQ(X&nR(]&vQ%O#^Q&y%PQ(k'SR)t(lpiO_ce#y$U$W%y%z't'z'})[*Y*[*x#]![Vdfmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r$V$]$^$f$m$|%O%|&y'c'x'|(](^)W)X*U*]*n+_+`+a+b+c+d+e+g+t+w+z,Y,[$PSOTV_cdefmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r#y$U$V$W$]$^$f$m$|%O%y%z%|&y'c't'x'z'|'}(](^)W)X)[*U*Y*[*]*n*x+_+`+a+b+c+d+e+g+t+w+z,Y,[RsRQrRR$s!|R!{qR$r!yQ$p!yR(Z&rQ#{!ZQ$b!nQ%{#|Q&a$cQ&c$eQ&l$jQ(P&bR(Y&o`#z!Z!n#|$c$e$j&b&o^%e#v'a'b(r)|*j+O!^,W$X$h%c%d%w&k&m&}'X'h'n(Q(X(c(n)d)j)n*P*e*s+Q+S+U+V+W+Y+[+|+}a,X'R(e+P+R+T+X+Z+]R%q#wQ%o#wR(z'eS%n#w'eS&f$g(Sb'Z%d'_'n+U+V+|+},Q,Rd'o%w(r)|*j+W+X+Y+Z+[+]S(_&})jQ(|'hR)e(R`#y!Z!n#|$c$e$j&b&oR%z#zR%v#xQ%t#xR)O'jQ!kgQ!qjS%s#x'jR&X$ZR$Y!jQ%k#vQ&W$XQ&j$hQ'T%cU'^%d+|+}d'o%w(r)|*j+W+X+Y+Z+[+]Q(V&kQ(W&mS(_&})jQ(g'RU(o'X+O+PQ(|'hU)Q'n+U+VQ)b(QQ)h(XQ)m(cQ)o(eQ)v(nQ*_)dQ*c)nQ*k*PQ*t*eR*y*sY%j#v(r)|*j+O!Y&V$X$h%c%d%w&k&m&}'X'h'n(Q(X(c(n)d)j)n*P*e*s+U+V+W+Y+[+|+}](j'R(e+P+X+Z+]Y%h#v(r)|*j+O!Y&U$X$h%c%d%w&k&m&}'X'h'n(Q(X(c(n)d)j)n*P*e*s+U+V+W+Y+[+|+}[(i'R(e+P+X+Z+]Q(x'bQ)_+SR*f+T!v%i#v$X$h%c%d%w&k&m&}'R'X'b'h'n(Q(X(c(e(n(r)d)j)n)|*P*e*j*s+O+P+S+T+U+V+W+X+Y+Z+[+]+|+}V(w'a+Q+R^%g#v'a'b(r)|*j+O!^&T$X$h%c%d%w&k&m&}'X'h'n(Q(X(c(n)d)j)n*P*e*s+Q+S+U+V+W+Y+[+|+}a(h'R(e+P+R+T+X+Z+]!}%f#v$X$h%c%d%w&k&m&}'R'X'a'b'h'n(Q(X(c(e(n(r)d)j)n)|*P*e*j*s+O+P+Q+R+S+T+U+V+W+X+Y+Z+[+]+|+}^%b#v'a'b(r)|*j+O!^&S$X$h%c%d%w&k&m&}'X'h'n(Q(X(c(n)d)j)n*P*e*s+Q+S+U+V+W+Y+[+|+}a(f'R(e+P+R+T+X+Z+]V'O%b&S(fR(b&}Q(`&}R*a)jd'o%w(r)|*j+W+X+Y+Z+[+]S(_&})jR)e(RX)S'n)d+U+Vb'Z%d'_'n+U+V+|+},Q,Rd'o%w(r)|*j+W+X+Y+Z+[+]S(_&})jQ)e(RR*k*PR'T%cQ'S%cV)s(k)t*gQ'Q%cV)r(k)t*gR'S%cS']%d+UQ(v'_S,O'n+|S,P+V+}Q,T,QR,U,Rb'Y%d'_'n+U+V+|+},Q,RR)x(pQ'q%wU){(r+W+XU*i)|+Y+ZV*u*j+[+]e'p%w(r)|*j+W+X+Y+Z+[+]T%s#x'jR'i%rR(}'hQ%y#yR't%zq{U|!R!V#d#h#t#u$O%S%^%_%}&{*}+^pYO_ce#y$U$W%y%z't'z'})[*Y*[*x#]!WVdfmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r$V$]$^$f$m$|%O%|&y'c'x'|(](^)W)X*U*]*n+_+`+a+b+c+d+e+g+t+w+z,Y,[Q!RUQ#d|S#u!V+^T%^#t*}[!PU|!V#t*}+^R%U#gQ!xmQ#f}Q$S!cQ&[$^R&d$fR!aXq[O_ce#y$U$W%y%z't'z'})[*Y*[*xQ!lgR!sjQ!nhR$c!oQ$i!tR&n$kR&i$gQ&g$gR)f(SR)b(RnaOce#y$U$W%y%z't'z'})[*Y*[*xR!e_q`O_ce#y$U$W%y%z't'z'})[*Y*[*xn_Oce#y$U$W%y%z't'z'})[*Y*[*xR!d_Q#QuR$u#QQ#TvR$w#TQ#XwR$y#XQ%P#^R&z%P#|TOV_cdefmxz}!U!]!a!c#U#[#k#l#m#n#o#q#r#y$U$V$W$]$^$f$m$|%O%y%z%|&y'c't'x'z'|'}(](^)W)X)[*U*Y*[*]*n*x+_+`+a+b+c+d+e+g+t+w+z,Y,[RtTQ&s$pR([&sQ!}rR$t!}Q'f%oR({'fQ$`!mU&^$`&|'{S&|%a)zX'{&R'P'W'mS*Q)Q*_R*l*QQ)k(`R*b)kQ(l'SR)u(lQ(q'YR)y(qQ'k%tR)P'kQ|U`#c|#h#t%S%_%}&{*}Q#h!RQ#t!VQ%S#dQ%_#uQ%}$OQ&{%^R*}+^Q$n!wS&q$n+hR+h+{Q)V'uS*T)V,VR,V,SQ!_XR$Q!_Q'w&QR)Y'wS$[!k!qR&Y$[Q(T&gR)g(T", + nodeNames: "⚠ LongString LongComment InterpStart InterpChunk InterpEnd InterpOpen InterpClose Space Newline LineComment Chunk Block Statement SimpleStatement VarList AssignableExpression PrimaryExpression Name Identifier ParenthesizedExpression ParenL Expression BinaryExpression or and CompareOp ShiftOp UnaryExpression UnaryOp not AsExpression SimpleExpression Number String StringContentDouble Escape StringContentSingle InterpolatedString InterpPart Interpolation nil true false TableConstructor BraceL FieldList Field BracketL BracketR AssignOp FieldSep BraceR FunctionExpression Attributes Attribute AttributeParamList AttributeParam AttributeArgs LiteralList Literal ParenR function FunctionBody TypeParams GenericTypeList GenericTypeParam GenericTypePack ParamList ParamListInner Param Binding TypeAnnotation TypeColon Type UnionType IntersectionType OptionalType SimpleType SingletonType typeof TypeReference TypeName TypeArgs TypeParamList TypeParam TypePack TypePackList VariadicTypePack TableType PropList TableProp ReadWrite read write TableIndexer FunctionType BoundTypeList BoundType TypeArrow ReturnTypeValue TypeOrPack ParenthesizedType VarArgParam VarArgAnnotation VarArgType ReturnType end FieldExpression FieldSuffix CallExpression CallSuffix FunctionArgs ExpList IfExpression if then elseif else TypeCast CompoundAssignOp CallStatement DoStatement do WhileStatement while RepeatStatement repeat until IfStatement ForNumericStatement for ForGenericStatement BindingList in FunctionDeclaration FunctionName LocalFunctionDeclaration local LocalDeclaration TypeDeclaration type TypeParamsWithDefaults GenericTypeListWithDefaults GenericTypeParamWithDefault TypePackDefault ExportTypeDeclaration export TypeFunctionDeclaration ExportTypeFunctionDeclaration LastStatement ReturnStatement return BreakStatement break ContinueStatement continue", + maxTerm: 221, + context: luauContext, + nodeProps: [ + ["isolate", -2,34,38,""] + ], + skippedNodes: [0,2,8,9,10], + repeatNodeCount: 22, + tokenData: "!3f~R!ROX$[XY&bYZ'mZ[&b[]&b]^'t^p$[pq&bqr$[rs(jst)Ttu$[uv)yvw+kwx,axy,zyz-pz{.f{|0W|}1x}!O2n!O!P8c!P!Q@V!Q!RCo!R![ET![!]Ko!]!^Ma!^!_NV!_!`!!p!`!a!$b!a!b!'w!b!c!(m!c!}!*[!}#O!+j#O#P!,`#P#Q!,z#Q#R!-r#R#S!*[#S#T$[#T#o!*[#o#p!/d#p#q!0Y#q#r!1O#r#s!1t#s$f$[$f$g&b$g;'S$[;'S;=`&[<%lO$[[$cXsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[W%TUuWOY%OZw%Ox#O%O#P;'S%O;'S;=`%g<%lO%OW%jP;=`<%l%OS%rUsSOY%mZr%ms#O%m#P;'S%m;'S;=`&U<%lO%mS&XP;=`<%l%m[&_P;=`<%l$[o&k`WcsSuWOX$[XY&bZ[&b[]&b]p$[pq&bqr$[rs%Osw$[wx%mx#O$[#P$f$[$f$g&b$g;'S$[;'S;=`&[<%lO$[o'tO%a[Xco'}XXcsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[o(qU%`guWOY%OZw%Ox#O%O#P;'S%O;'S;=`%g<%lO%O^)^X%^QsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[m*SZ%[`sSuWOY$[Zr$[rs%Osw$[wx%mx!_$[!_!`*u!`#O$[#P;'S$[;'S;=`&[<%lO$[]+OX%sPsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[o+tX%kcsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[o,hU%bksSOY%mZr%ms#O%m#P;'S%m;'S;=`&U<%lO%mo-TXecsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[o-yX!_csSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[m.oZ%X`sSuWOY$[Zr$[rs%Osw$[wx%mx!_$[!_!`/b!`#O$[#P;'S$[;'S;=`&[<%lO$[]/kX%pPsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[m0aZsSuW%V`OY$[Zr$[rs%Osw$[wx%mx!_$[!_!`1S!`#O$[#P;'S$[;'S;=`&[<%lO$[]1]XsSuW%nPOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[m2RX%easSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[o2w^sSuW%WbOY$[Zr$[rs%Osw$[wx%mx}$[}!O3s!O!_$[!_!`6w!`!a7m!a#O$[#P;'S$[;'S;=`&[<%lO$[o3|YsSuWYcOY3sZr3srs4lsw3swx5zx#O3s#O#P5]#P;'S3s;'S;=`6q<%lO3sk4sWuWYcOY4lZw4lwx5]x#O4l#O#P5]#P;'S4l;'S;=`5t<%lO4lc5bSYcOY5]Z;'S5];'S;=`5n<%lO5]c5qP;=`<%l5]k5wP;=`<%l4lg6RWsSYcOY5zZr5zrs5]s#O5z#O#P5]#P;'S5z;'S;=`6k<%lO5zg6nP;=`<%l5zo6tP;=`<%l3s]7QXsSuW%oPOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[o7vXsSuW#VcOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[o8l]sSuW%laOY$[Zr$[rs%Osw$[wx%mx!O$[!O!P9e!P!Q$[!Q![a|}$[}!O>a!O!Q$[!Q![?Z![#O$[#P;'S$[;'S;=`&[<%lO$[^>hZsSuWOY$[Zr$[rs%Osw$[wx%mx!Q$[!Q![?Z![#O$[#P;'S$[;'S;=`&[<%lO$[^?dZqQsSuWOY$[Zr$[rs%Osw$[wx%mx!Q$[!Q![?Z![#O$[#P;'S$[;'S;=`&[<%lO$[m@`]sSuW%Y`OY$[Zr$[rs%Osw$[wx%mx!P$[!P!QAX!Q!_$[!_!`By!`#O$[#P;'S$[;'S;=`&[<%lO$[mAbZsSuW%Z`OY$[Zr$[rs%Osw$[wx%mx!_$[!_!`BT!`#O$[#P;'S$[;'S;=`&[<%lO$[]B^XsSuW%rPOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[]CSX%qPsSuWOY$[Zr$[rs%Osw$[wx%mx#O$[#P;'S$[;'S;=`&[<%lO$[^CxcqQsSuWOY$[Zr$[rs%Osw$[wx%mx!O$[!O!P spec_Identifier[value] || -1}], + tokenPrec: 6676 +}) diff --git a/src/lib/codemirror-lang-luau/parser.terms.js b/src/lib/codemirror-lang-luau/parser.terms.js new file mode 100644 index 0000000..6489c30 --- /dev/null +++ b/src/lib/codemirror-lang-luau/parser.terms.js @@ -0,0 +1,159 @@ +// @ts-nocheck +// This file was generated by lezer-generator. You probably shouldn't edit it. +export const + LongString = 1, + LongComment = 2, + InterpStart = 3, + InterpChunk = 4, + InterpEnd = 5, + InterpOpen = 6, + InterpClose = 7, + Space = 8, + Newline = 9, + LineComment = 10, + Chunk = 11, + Block = 12, + Statement = 13, + SimpleStatement = 14, + VarList = 15, + AssignableExpression = 16, + PrimaryExpression = 17, + Name = 18, + Identifier = 19, + ParenthesizedExpression = 20, + ParenL = 21, + Expression = 22, + BinaryExpression = 23, + or = 24, + and = 25, + CompareOp = 26, + ShiftOp = 27, + UnaryExpression = 28, + UnaryOp = 29, + not = 30, + AsExpression = 31, + SimpleExpression = 32, + Number = 33, + String = 34, + StringContentDouble = 35, + Escape = 36, + StringContentSingle = 37, + InterpolatedString = 38, + InterpPart = 39, + Interpolation = 40, + nil = 41, + _true = 42, + _false = 43, + TableConstructor = 44, + BraceL = 45, + FieldList = 46, + Field = 47, + BracketL = 48, + BracketR = 49, + AssignOp = 50, + FieldSep = 51, + BraceR = 52, + FunctionExpression = 53, + Attributes = 54, + Attribute = 55, + AttributeParamList = 56, + AttributeParam = 57, + AttributeArgs = 58, + LiteralList = 59, + Literal = 60, + ParenR = 61, + _function = 62, + FunctionBody = 63, + TypeParams = 64, + GenericTypeList = 65, + GenericTypeParam = 66, + GenericTypePack = 67, + ParamList = 68, + ParamListInner = 69, + Param = 70, + Binding = 71, + TypeAnnotation = 72, + TypeColon = 73, + Type = 74, + UnionType = 75, + IntersectionType = 76, + OptionalType = 77, + SimpleType = 78, + SingletonType = 79, + _typeof = 80, + TypeReference = 81, + TypeName = 82, + TypeArgs = 83, + TypeParamList = 84, + TypeParam = 85, + TypePack = 86, + TypePackList = 87, + VariadicTypePack = 88, + TableType = 89, + PropList = 90, + TableProp = 91, + ReadWrite = 92, + read = 93, + write = 94, + TableIndexer = 95, + FunctionType = 96, + BoundTypeList = 97, + BoundType = 98, + TypeArrow = 99, + ReturnTypeValue = 100, + TypeOrPack = 101, + ParenthesizedType = 102, + VarArgParam = 103, + VarArgAnnotation = 104, + VarArgType = 105, + ReturnType = 106, + end = 107, + FieldExpression = 108, + FieldSuffix = 109, + CallExpression = 110, + CallSuffix = 111, + FunctionArgs = 112, + ExpList = 113, + IfExpression = 114, + _if = 115, + then = 116, + elseif = 117, + _else = 118, + TypeCast = 119, + CompoundAssignOp = 120, + CallStatement = 121, + DoStatement = 122, + _do = 123, + WhileStatement = 124, + _while = 125, + RepeatStatement = 126, + repeat = 127, + until = 128, + IfStatement = 129, + ForNumericStatement = 130, + _for = 131, + ForGenericStatement = 132, + BindingList = 133, + _in = 134, + FunctionDeclaration = 135, + FunctionName = 136, + LocalFunctionDeclaration = 137, + local = 138, + LocalDeclaration = 139, + TypeDeclaration = 140, + type = 141, + TypeParamsWithDefaults = 142, + GenericTypeListWithDefaults = 143, + GenericTypeParamWithDefault = 144, + TypePackDefault = 145, + ExportTypeDeclaration = 146, + _export = 147, + TypeFunctionDeclaration = 148, + ExportTypeFunctionDeclaration = 149, + LastStatement = 150, + ReturnStatement = 151, + _return = 152, + BreakStatement = 153, + _break = 154, + ContinueStatement = 155, + _continue = 156 diff --git a/src/lib/codemirror-lang-luau/tokenizer.js b/src/lib/codemirror-lang-luau/tokenizer.js new file mode 100644 index 0000000..5da7f93 --- /dev/null +++ b/src/lib/codemirror-lang-luau/tokenizer.js @@ -0,0 +1,148 @@ +import { ExternalTokenizer, ContextTracker } from "@lezer/lr"; +import { + BraceL, + BraceR, + InterpChunk, + InterpClose, + InterpEnd, + InterpOpen, + InterpStart, + LongComment, + LongString, +} from "./parser.terms.js"; + +const dash = 45; +const backtick = 96; +const backslash = 92; +const bracketL = 91; +const bracketR = 93; +const braceL = 123; +const braceR = 125; +const eq = 61; + +function isInInterpolatedString(context) { + return context && (context.inString || context.inInterpolation); +} + +function scanLongBracket(input, eqCount) { + while (input.next > -1) { + if (input.next == bracketR) { + let offset = 1; + while (input.peek(offset) == eq) offset++; + if (offset - 1 == eqCount && input.peek(offset) == bracketR) { + input.advance(); + for (let i = 0; i < eqCount; i++) input.advance(); + input.advance(); + return; + } + } + input.advance(); + } +} + +function matchLongBracketStart(input, startOffset) { + let offset = startOffset; + let eqCount = 0; + while (input.peek(offset) == eq) { + eqCount++; + offset++; + } + if (input.peek(offset) != bracketL) return null; + return { eqCount, endOffset: offset }; +} + +export const longBracket = new ExternalTokenizer((input, stack) => { + if (isInInterpolatedString(stack.context)) return; + + if (input.next == dash && input.peek(1) == dash && input.peek(2) == bracketL) { + const match = matchLongBracketStart(input, 3); + if (!match) return; + + input.advance(); + input.advance(); + input.advance(); + for (let i = 0; i < match.eqCount; i++) input.advance(); + input.advance(); + + scanLongBracket(input, match.eqCount); + input.acceptToken(LongComment); + return; + } + + if (input.next != bracketL) return; + const match = matchLongBracketStart(input, 1); + if (!match) return; + + input.advance(); + for (let i = 0; i < match.eqCount; i++) input.advance(); + input.advance(); + + scanLongBracket(input, match.eqCount); + input.acceptToken(LongString); +}, { contextual: true }); + +export const luauContext = new ContextTracker({ + start: { inString: false, inInterpolation: false, braceDepth: 0 }, + shift(context, term) { + if (term == InterpStart) return { inString: true, inInterpolation: false, braceDepth: 0 }; + if (term == InterpEnd) return { inString: false, inInterpolation: false, braceDepth: 0 }; + if (term == InterpOpen) return { inString: false, inInterpolation: true, braceDepth: 0 }; + if (term == InterpClose) return { inString: true, inInterpolation: false, braceDepth: 0 }; + + if (context.inInterpolation) { + if (term == BraceL) return { ...context, braceDepth: context.braceDepth + 1 }; + if (term == BraceR && context.braceDepth > 0) { + return { ...context, braceDepth: context.braceDepth - 1 }; + } + } + + return context; + }, + strict: false, +}); + +export const backtickTokens = new ExternalTokenizer( + (input, stack) => { + const context = stack.context; + if (!context || (!context.inString && !context.inInterpolation)) { + if (input.next != backtick) return; + input.advance(); + input.acceptToken(InterpStart); + return; + } + + if (context.inString) { + if (input.next == backtick) { + input.advance(); + input.acceptToken(InterpEnd); + return; + } + if (input.next == braceL) { + input.advance(); + input.acceptToken(InterpOpen); + return; + } + + let size = 0; + while (input.next > -1 && input.next != backtick && input.next != braceL) { + if (input.next == backslash) { + input.advance(); + size++; + if (input.next < 0) break; + } + input.advance(); + size++; + } + if (size > 0) input.acceptToken(InterpChunk); + return; + } + + if (context.inInterpolation) { + if (input.next == braceR && context.braceDepth == 0 && stack.canShift(InterpClose)) { + input.advance(); + input.acceptToken(InterpClose); + } + } + }, + { contextual: true } +); From 004e429ef0a0937466cd24643fd6d77481340fa9 Mon Sep 17 00:00:00 2001 From: Nick Winans Date: Mon, 5 Jan 2026 20:59:47 -0800 Subject: [PATCH 2/3] use lezer parser --- src/lib/components/Editor.svelte | 4 ---- src/lib/editor/index.ts | 1 - src/lib/editor/setup.ts | 5 ++--- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/lib/components/Editor.svelte b/src/lib/components/Editor.svelte index dfefdcb..0bfaa8b 100644 --- a/src/lib/components/Editor.svelte +++ b/src/lib/components/Editor.svelte @@ -13,9 +13,6 @@ onMount(async () => { editorModule = await import('$lib/editor/setup'); - // Initialize TextMate grammar (no WASM, resolves immediately) - await editorModule.initLuauTextMate(); - editorModule.createEditor(editorContainer, $files[$activeFile] || '', (content) => { if ($activeFile) { updateFile($activeFile, content); @@ -72,4 +69,3 @@ {/if} - diff --git a/src/lib/editor/index.ts b/src/lib/editor/index.ts index 12480e4..127bcd6 100644 --- a/src/lib/editor/index.ts +++ b/src/lib/editor/index.ts @@ -1,4 +1,3 @@ export { createEditor, destroyEditor, updateEditorContent, getEditorContent, getEditorView, focusEditor } from './setup'; -export { luauTextMate, initLuauTextMate } from './textmate'; export { darkTheme, lightTheme } from './themes'; export { luauLspExtensions, createLuauLinter, createLuauAutocomplete, createLuauHover } from './lspExtensions'; diff --git a/src/lib/editor/setup.ts b/src/lib/editor/setup.ts index b3997b1..6ce34db 100644 --- a/src/lib/editor/setup.ts +++ b/src/lib/editor/setup.ts @@ -12,8 +12,7 @@ import { bracketMatching, indentOnInput } from '@codemirror/language'; import { closeBrackets, closeBracketsKeymap } from '@codemirror/autocomplete'; import { highlightSelectionMatches, searchKeymap } from '@codemirror/search'; -import { luauTextMate, initLuauTextMate } from './textmate'; -export { initLuauTextMate }; +import { luau } from '$lib/codemirror-lang-luau/index.js'; import { darkTheme, lightTheme } from './themes'; import { luauLspExtensions } from './lspExtensions'; import { forceLinting, lintGutter } from '@codemirror/lint'; @@ -67,7 +66,7 @@ function createExtensions(onChange: (content: string) => void): Extension[] { ]), // Luau language + LSP extensions - luauTextMate(), + luau(), ...luauLspExtensions(), // Theme (dynamic) From 7f80a182f77d5e81094f5857993879122ad56639 Mon Sep 17 00:00:00 2001 From: Nick Winans Date: Mon, 5 Jan 2026 23:05:56 -0800 Subject: [PATCH 3/3] drop textmate --- src/lib/editor/Luau.tmLanguage.json | 856 ---------------------------- src/lib/editor/js-regex-engine.ts | 122 ---- src/lib/editor/textmate.ts | 217 ------- vite.config.ts | 3 +- vite/compile-grammar.ts | 180 ------ vite/prerender.ts | 3 +- 6 files changed, 2 insertions(+), 1379 deletions(-) delete mode 100644 src/lib/editor/Luau.tmLanguage.json delete mode 100644 src/lib/editor/js-regex-engine.ts delete mode 100644 src/lib/editor/textmate.ts delete mode 100644 vite/compile-grammar.ts diff --git a/src/lib/editor/Luau.tmLanguage.json b/src/lib/editor/Luau.tmLanguage.json deleted file mode 100644 index 2a4d0b4..0000000 --- a/src/lib/editor/Luau.tmLanguage.json +++ /dev/null @@ -1,856 +0,0 @@ -{ - "information_for_contributors": [ - "The source of this file is https://github.com/JohnnyMorganz/Luau.tmLanguage/blob/main/Luau.YAML-tmLanguage", - "If you want to provide a fix or improvement, please create a pull request against the original repository.", - "Once accepted there, we are happy to receive an update request." - ], - "name": "Luau", - "scopeName": "source.luau", - "fileTypes": [ - "luau" - ], - "patterns": [ - { - "include": "#function-definition" - }, - { - "include": "#number" - }, - { - "include": "#string" - }, - { - "include": "#shebang" - }, - { - "include": "#comment" - }, - { - "include": "#local-declaration" - }, - { - "include": "#for-loop" - }, - { - "include": "#type-function" - }, - { - "include": "#type-alias-declaration" - }, - { - "include": "#keyword" - }, - { - "include": "#language_constant" - }, - { - "include": "#standard_library" - }, - { - "include": "#identifier" - }, - { - "include": "#operator" - }, - { - "include": "#parentheses" - }, - { - "include": "#table" - }, - { - "include": "#type_cast" - }, - { - "include": "#type_annotation" - }, - { - "include": "#attribute" - } - ], - "repository": { - "function-definition": { - "begin": "\\b(?:(local)\\s+)?(function)\\b(?![,:])", - "beginCaptures": { - "1": { - "name": "storage.modifier.local.luau" - }, - "2": { - "name": "keyword.control.luau" - } - }, - "end": "(?<=[\\)\\-{}\\[\\]\"'])", - "name": "meta.function.luau", - "patterns": [ - { - "include": "#comment" - }, - { - "include": "#generics-declaration" - }, - { - "begin": "(\\()", - "beginCaptures": { - "1": { - "name": "punctuation.definition.parameters.begin.luau" - } - }, - "end": "(\\))", - "endCaptures": { - "1": { - "name": "punctuation.definition.parameters.end.luau" - } - }, - "name": "meta.parameter.luau", - "patterns": [ - { - "include": "#comment" - }, - { - "match": "\\.\\.\\.", - "name": "variable.parameter.function.varargs.luau" - }, - { - "match": "[a-zA-Z_][a-zA-Z0-9_]*", - "name": "variable.parameter.function.luau" - }, - { - "match": ",", - "name": "punctuation.separator.arguments.luau" - }, - { - "begin": ":", - "beginCaptures": { - "0": { - "name": "keyword.operator.type.luau" - } - }, - "end": "(?=[\\),])", - "patterns": [ - { - "include": "#type_literal" - } - ] - } - ] - }, - { - "match": "\\b(__add|__call|__concat|__div|__eq|__index|__le|__len|__lt|__metatable|__mod|__mode|__mul|__newindex|__pow|__sub|__tostring|__unm|__iter|__idiv)\\b", - "name": "variable.language.metamethod.luau" - }, - { - "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b", - "name": "entity.name.function.luau" - } - ] - }, - "local-declaration": { - "begin": "\\b(local)\\b", - "end": "(?=\\s*do\\b|\\s*[=;]|\\s*$)", - "beginCaptures": { - "1": { - "name": "storage.modifier.local.luau" - } - }, - "patterns": [ - { - "include": "#comment" - }, - { - "include": "#attribute" - }, - { - "begin": "(:)", - "beginCaptures": { - "1": { - "name": "keyword.operator.type.luau" - } - }, - "end": "(?=\\s*do\\b|\\s*[=;,]|\\s*$)", - "patterns": [ - { - "include": "#type_literal" - } - ] - }, - { - "name": "variable.other.constant.luau", - "match": "\\b([A-Z_][A-Z0-9_]*)\\b" - }, - { - "name": "variable.other.readwrite.luau", - "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b" - } - ] - }, - "for-loop": { - "begin": "\\b(for)\\b", - "beginCaptures": { - "1": { - "name": "keyword.control.luau" - } - }, - "end": "\\b(in)\\b|(=)", - "endCaptures": { - "1": { - "name": "keyword.control.luau" - }, - "2": { - "name": "keyword.operator.assignment.luau" - } - }, - "patterns": [ - { - "begin": "(:)", - "beginCaptures": { - "1": { - "name": "keyword.operator.type.luau" - } - }, - "end": "(?=\\s*in\\b|\\s*[=,]|\\s*$)", - "patterns": [ - { - "include": "#type_literal" - } - ] - }, - { - "name": "variable.parameter.luau", - "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b" - } - ] - }, - "shebang": { - "captures": { - "1": { - "name": "punctuation.definition.comment.luau" - } - }, - "match": "\\A(#!).*$\\n?", - "name": "comment.line.shebang.luau" - }, - "string_escape": { - "patterns": [ - { - "name": "constant.character.escape.luau", - "match": "\\\\[abfnrtvz'\"`{\\\\]" - }, - { - "name": "constant.character.escape.luau", - "match": "\\\\\\d{1,3}" - }, - { - "name": "constant.character.escape.luau", - "match": "\\\\x[0-9a-fA-F]{2}" - }, - { - "name": "constant.character.escape.luau", - "match": "\\\\u\\{[0-9a-fA-F]*\\}" - }, - { - "name": "constant.character.escape.luau", - "match": "\\\\$" - } - ] - }, - "number": { - "patterns": [ - { - "name": "constant.numeric.hex.luau", - "match": "\\b0_*[xX]_*[\\da-fA-F_]*(?:[eE][\\+\\-]?_*\\d[\\d_]*(?:\\.[\\d_]*)?)?" - }, - { - "name": "constant.numeric.binary.luau", - "match": "\\b0_*[bB][01_]+(?:[eE][\\+\\-]?_*\\d[\\d_]*(?:\\.[\\d_]*)?)?" - }, - { - "name": "constant.numeric.decimal.luau", - "match": "(?:\\d[\\d_]*(?:\\.[\\d_]*)?|\\.\\d[\\d_]*)(?:[eE][\\+\\-]?_*\\d[\\d_]*(?:\\.[\\d_]*)?)?" - } - ] - }, - "string": { - "patterns": [ - { - "name": "string.quoted.double.luau", - "begin": "\"", - "end": "\"", - "patterns": [ - { - "include": "#string_escape" - } - ] - }, - { - "name": "string.quoted.single.luau", - "begin": "'", - "end": "'", - "patterns": [ - { - "include": "#string_escape" - } - ] - }, - { - "name": "string.other.multiline.luau", - "begin": "\\[(=*)\\[", - "end": "\\]\\1\\]" - }, - { - "name": "string.interpolated.luau", - "begin": "`", - "end": "`", - "patterns": [ - { - "include": "#interpolated_string_expression" - }, - { - "include": "#string_escape" - } - ] - } - ] - }, - "interpolated_string_expression": { - "name": "meta.template.expression.luau", - "contentName": "meta.embedded.line.luau", - "begin": "\\{", - "end": "\\}", - "beginCaptures": { - "0": { - "name": "punctuation.definition.interpolated-string-expression.begin.luau" - } - }, - "endCaptures": { - "0": { - "name": "punctuation.definition.interpolated-string-expression.end.luau" - } - }, - "patterns": [ - { - "include": "source.luau" - } - ] - }, - "standard_library": { - "patterns": [ - { - "name": "support.function.luau", - "match": "(?=?", - "name": "keyword.operator.comparison.luau" - }, - { - "match": "\\+=|-=|/=|//=|\\*=|%=|\\^=|\\.\\.=|=", - "name": "keyword.operator.assignment.luau" - }, - { - "match": "\\+|-|%|\\*|\\/\\/|\\/|\\^", - "name": "keyword.operator.arithmetic.luau" - }, - { - "match": "#|(?)", - "patterns": [ - { - "match": "[a-zA-Z_][a-zA-Z0-9_]*", - "name": "entity.name.type.luau" - }, - { - "match": "=", - "name": "keyword.operator.assignment.luau" - }, - { - "include": "#type_literal" - } - ] - }, - "type-function": { - "begin": "^\\b(?:(export)\\s+)?(type)\\s+(function)\\b", - "beginCaptures": { - "1": { - "name": "storage.modifier.visibility.luau" - }, - "2": { - "name": "storage.type.luau" - }, - "3": { - "name": "keyword.control.luau" - } - }, - "end": "(?<=[\\)\\-{}\\[\\]\"'])", - "name": "meta.function.luau", - "patterns": [ - { - "include": "#comment" - }, - { - "include": "#generics-declaration" - }, - { - "begin": "(\\()", - "beginCaptures": { - "1": { - "name": "punctuation.definition.parameters.begin.luau" - } - }, - "end": "(\\))", - "endCaptures": { - "1": { - "name": "punctuation.definition.parameters.end.luau" - } - }, - "name": "meta.parameter.luau", - "patterns": [ - { - "include": "#comment" - }, - { - "match": "\\.\\.\\.", - "name": "variable.parameter.function.varargs.luau" - }, - { - "match": "[a-zA-Z_][a-zA-Z0-9_]*", - "name": "variable.parameter.function.luau" - }, - { - "match": ",", - "name": "punctuation.separator.arguments.luau" - }, - { - "begin": ":", - "beginCaptures": { - "0": { - "name": "keyword.operator.type.luau" - } - }, - "end": "(?=[\\),])", - "patterns": [ - { - "include": "#type_literal" - } - ] - } - ] - }, - { - "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b", - "name": "entity.name.type.luau" - } - ] - }, - "type-alias-declaration": { - "begin": "^\\b(?:(export)\\s+)?(type)\\b", - "end": "(?=\\s*$)|(?=\\s*;)", - "beginCaptures": { - "1": { - "name": "storage.modifier.visibility.luau" - }, - "2": { - "name": "storage.type.luau" - } - }, - "patterns": [ - { - "include": "#type_literal" - }, - { - "match": "=", - "name": "keyword.operator.assignment.luau" - } - ] - }, - "type_annotation": { - "begin": ":(?!\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b(?=\\s*(?:[({\"']|\\[\\[)))", - "end": "(?<=\\))(?!\\s*->)|=|;|$|(?=\\breturn\\b)|(?=\\bend\\b)", - "patterns": [ - { - "include": "#comment" - }, - { - "include": "#type_literal" - } - ] - }, - "type_cast": { - "begin": "(::)", - "beginCaptures": { - "1": { - "name": "keyword.operator.typecast.luau" - } - }, - "end": "(?=^|[;),}\\]:?\\-\\+\\>](?!\\s*[&\\|])|$|\\b(break|do|else|for|if|elseif|return|then|repeat|while|until|end|in|continue)\\b)", - "patterns": [ - { - "include": "#type_literal" - } - ] - }, - "type_literal": { - "patterns": [ - { - "include": "#comment" - }, - { - "include": "#string" - }, - { - "match": "\\?|\\&|\\|", - "name": "keyword.operator.type.luau" - }, - { - "match": "->", - "name": "keyword.operator.type.function.luau" - }, - { - "match": "\\b(false)\\b", - "name": "constant.language.boolean.false.luau" - }, - { - "match": "\\b(true)\\b", - "name": "constant.language.boolean.true.luau" - }, - { - "match": "\\b(nil|string|number|boolean|thread|userdata|symbol|vector|buffer|unknown|never|any)\\b", - "name": "support.type.primitive.luau" - }, - { - "begin": "\\b(typeof)\\b(\\()", - "beginCaptures": { - "1": { - "name": "support.function.luau" - }, - "2": { - "name": "punctuation.arguments.begin.typeof.luau" - } - }, - "end": "(\\))", - "endCaptures": { - "1": { - "name": "punctuation.arguments.end.typeof.luau" - } - }, - "patterns": [ - { - "include": "source.luau" - } - ] - }, - { - "begin": "(<)", - "end": "(>)", - "beginCaptures": { - "1": { - "name": "punctuation.definition.typeparameters.begin.luau" - } - }, - "endCaptures": { - "1": { - "name": "punctuation.definition.typeparameters.end.luau" - } - }, - "patterns": [ - { - "match": "=", - "name": "keyword.operator.assignment.luau" - }, - { - "include": "#type_literal" - } - ] - }, - { - "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b", - "name": "entity.name.type.luau" - }, - { - "begin": "\\{", - "end": "\\}", - "patterns": [ - { - "begin": "\\[", - "end": "\\]", - "patterns": [ - { - "include": "#type_literal" - } - ] - }, - { - "match": "\\b(?:(read|write)\\s+)?([a-zA-Z_][a-zA-Z0-9_]*)\\b(:)", - "captures": { - "1": { - "name": "storage.modifier.access.luau" - }, - "2": { - "name": "variable.property.luau" - }, - "3": { - "name": "keyword.operator.type.luau" - } - } - }, - { - "include": "#type_literal" - }, - { - "match": "[,;]", - "name": "punctuation.separator.fields.type.luau" - } - ] - }, - { - "begin": "\\(", - "end": "\\)", - "patterns": [ - { - "name": "variable.parameter.luau", - "match": "\\b([a-zA-Z_][a-zA-Z0-9_]*)\\b(:)", - "captures": { - "1": { - "name": "variable.parameter.luau" - }, - "2": { - "name": "keyword.operator.type.luau" - } - } - }, - { - "include": "#type_literal" - } - ] - } - ] - }, - "attribute": { - "patterns": [ - { - "name": "meta.attribute.luau", - "match": "(@)([a-zA-Z_][a-zA-Z0-9_]*)", - "captures": { - "1": { - "name": "keyword.operator.attribute.luau" - }, - "2": { - "name": "storage.type.attribute.luau" - } - } - } - ] - } - } -} \ No newline at end of file diff --git a/src/lib/editor/js-regex-engine.ts b/src/lib/editor/js-regex-engine.ts deleted file mode 100644 index 799f47f..0000000 --- a/src/lib/editor/js-regex-engine.ts +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Minimal JavaScript regex engine for vscode-textmate. - * Uses pre-compiled patterns from compile-grammar.ts vite plugin - * - * This eliminates the need for oniguruma-to-es, vscode-oniguruma and oniguruma WASM at runtime. - */ - -import { compiledPatterns } from 'virtual:compiled-patterns'; - -const MAX = 4294967295; - -interface CaptureIndex { - start: number; - end: number; - length: number; -} - -interface Match { - index: number; - captureIndices: CaptureIndex[]; -} - -interface OnigString { - content: string; -} - -/** - * JavaScript-based scanner compatible with vscode-textmate's IOnigScanner interface. - */ -class JavaScriptScanner { - private regexps: (RegExp | null)[]; - - constructor(patterns: string[]) { - this.regexps = patterns.map((pattern) => { - const compiled = compiledPatterns[pattern]; - if (!compiled) { - // Pattern not in cache - this shouldn't happen for Luau grammar - console.warn(`[JS Scanner] Pattern not pre-compiled: ${pattern.slice(0, 50)}`); - return null; - } - try { - return new RegExp(compiled[0], compiled[1]); - } catch (e) { - console.warn(`[JS Scanner] Failed to construct RegExp: ${pattern.slice(0, 50)}`); - return null; - } - }); - } - - findNextMatchSync( - string: string | OnigString, - startPosition: number - ): Match | null { - const str = typeof string === 'string' ? string : string.content; - const pending: [number, RegExpExecArray][] = []; - - for (let i = 0; i < this.regexps.length; i++) { - const regexp = this.regexps[i]; - if (!regexp) continue; - - try { - regexp.lastIndex = startPosition; - const match = regexp.exec(str); - if (!match) continue; - - // If match starts at startPosition, return immediately - if (match.index === startPosition) { - return this.toResult(i, match); - } - pending.push([i, match]); - } catch { - continue; - } - } - - // Return the match with the earliest start position - if (pending.length) { - const minIndex = Math.min(...pending.map((m) => m[1].index)); - for (const [i, match] of pending) { - if (match.index === minIndex) { - return this.toResult(i, match); - } - } - } - - return null; - } - - private toResult(index: number, match: RegExpExecArray): Match { - return { - index, - captureIndices: (match.indices || []).map((indice) => { - if (indice == null) { - return { start: MAX, end: MAX, length: 0 }; - } - return { - start: indice[0], - end: indice[1], - length: indice[1] - indice[0], - }; - }), - }; - } -} - -/** - * OnigString implementation (just wraps the string). - */ -class OnigStringImpl implements OnigString { - constructor(public content: string) {} -} - -/** - * Create the onigLib object expected by vscode-textmate Registry. - */ -export function createJsOnigLib() { - return Promise.resolve({ - createOnigScanner: (patterns: string[]) => new JavaScriptScanner(patterns), - createOnigString: (str: string) => new OnigStringImpl(str), - }); -} - diff --git a/src/lib/editor/textmate.ts b/src/lib/editor/textmate.ts deleted file mode 100644 index 901ddf1..0000000 --- a/src/lib/editor/textmate.ts +++ /dev/null @@ -1,217 +0,0 @@ -/** - * TextMate Grammar Integration for CodeMirror 6 - * - * Uses vscode-textmate with the official Luau.tmLanguage grammar - * from https://github.com/JohnnyMorganz/Luau.tmLanguage - * - * Regex patterns are pre-compiled by compile-grammar.ts vite plugin - * This eliminates the need for oniguruma WASM at runtime. - */ - -import { StreamLanguage, LanguageSupport, type StringStream } from '@codemirror/language'; -import * as vsctm from 'vscode-textmate'; -import { createJsOnigLib } from './js-regex-engine'; -import luauGrammar from './Luau.tmLanguage.json'; - -// Singleton state -let registry: vsctm.Registry | null = null; -let grammar: vsctm.IGrammar | null = null; -let initPromise: Promise | null = null; - -/** - * Initialize the TextMate registry and load the Luau grammar. - */ -async function initTextMate(): Promise { - if (grammar) return; - - if (initPromise) { - await initPromise; - return; - } - - initPromise = (async () => { - // Grammar is bundled - convert to string for parsing - const grammarJson = JSON.stringify(luauGrammar); - - // Create the registry with JS-based regex engine (no WASM needed) - registry = new vsctm.Registry({ - onigLib: createJsOnigLib(), - loadGrammar: async (scopeName) => { - if (scopeName === 'source.luau') { - return vsctm.parseRawGrammar(grammarJson, 'Luau.tmLanguage.json'); - } - return null; - }, - }); - - // Load the grammar - grammar = await registry.loadGrammar('source.luau'); - - if (!grammar) { - throw new Error('Failed to load Luau grammar'); - } - - console.log('[TextMate] Luau grammar loaded successfully (JS regex engine)'); - })(); - - await initPromise; -} - -/** - * Map TextMate scopes to CodeMirror token types. - */ -function scopeToToken(scopes: string[]): string | null { - // Check scopes from most specific to least specific - for (let i = scopes.length - 1; i >= 0; i--) { - const scope = scopes[i]; - - // Comments - if (scope.startsWith('comment')) return 'comment'; - - // Strings - if (scope.startsWith('string')) return 'string'; - - // Numbers - if (scope.startsWith('constant.numeric')) return 'number'; - - // Booleans and nil - if (scope.includes('constant.language.boolean')) return 'bool'; - if (scope.includes('constant.language.nil')) return 'null'; - if (scope.startsWith('constant.language')) return 'atom'; - if (scope.startsWith('constant')) return 'atom'; - - // Keywords - if (scope.startsWith('keyword.control')) return 'keyword'; - if (scope.startsWith('keyword.operator')) return 'operator'; - if (scope.startsWith('keyword')) return 'keyword'; - - // Storage (local, function, etc.) - if (scope.startsWith('storage')) return 'keyword'; - - // Types - if (scope.startsWith('entity.name.type')) return 'typeName'; - if (scope.startsWith('support.type')) return 'typeName'; - - // Functions - if (scope.startsWith('entity.name.function')) return 'variableName.function'; - if (scope.startsWith('support.function')) return 'variableName.function'; - if (scope.includes('function-call')) return 'variableName.function'; - - // Variables - if (scope.startsWith('variable.parameter')) return 'variableName.definition'; - if (scope.startsWith('variable')) return 'variableName'; - - // Operators - if (scope.startsWith('keyword.operator')) return 'operator'; - - // Punctuation - if (scope.startsWith('punctuation')) return 'punctuation'; - - // Support (built-in functions/types) - if (scope.startsWith('support')) return 'variableName.standard'; - } - - return null; -} - -/** - * TextMate tokenizer state for CodeMirror. - */ -interface TMState { - ruleStack: vsctm.StateStack; - lineTokens: Array<{ startIndex: number; endIndex: number; scopes: string[] }> | null; - lineText: string; -} - -/** - * Create a CodeMirror StreamLanguage that uses TextMate tokenization. - */ -function createTextMateLanguage(): StreamLanguage { - return StreamLanguage.define({ - name: 'luau', - - startState: (): TMState => ({ - ruleStack: vsctm.INITIAL, - lineTokens: null, - lineText: '', - }), - - copyState: (state: TMState): TMState => ({ - ruleStack: state.ruleStack, - lineTokens: state.lineTokens, - lineText: state.lineText, - }), - - token: (stream: StringStream, state: TMState): string | null => { - if (!grammar) { - // Grammar not loaded yet, consume the line - stream.skipToEnd(); - return null; - } - - // At the start of a line, tokenize the entire line - if (stream.sol()) { - const lineText = stream.string; - const result = grammar.tokenizeLine(lineText, state.ruleStack); - - state.ruleStack = result.ruleStack; - state.lineText = lineText; - - // Convert tokens to our format - state.lineTokens = result.tokens.map((t, i, arr) => ({ - startIndex: t.startIndex, - endIndex: i < arr.length - 1 ? arr[i + 1].startIndex : lineText.length, - scopes: t.scopes, - })); - } - - // Find the token at the current position - const pos = stream.pos; - const tokens = state.lineTokens || []; - - for (const token of tokens) { - if (pos >= token.startIndex && pos < token.endIndex) { - // Consume this token - stream.pos = token.endIndex; - return scopeToToken(token.scopes); - } - } - - // No token found, consume one character - stream.next(); - return null; - }, - - languageData: { - commentTokens: { line: '--', block: { open: '--[[', close: ']]' } }, - closeBrackets: { brackets: ['(', '[', '{', '"', "'"] }, - indentOnInput: /^\s*(end|else|elseif|until|\}|\])$/, - }, - }); -} - -// Cached language instance -let textMateLanguage: StreamLanguage | null = null; - -/** - * Get or create the TextMate-based Luau language. - * Returns a fallback if TextMate isn't loaded yet. - */ -export function luauTextMate(): LanguageSupport { - if (!textMateLanguage) { - textMateLanguage = createTextMateLanguage(); - } - return new LanguageSupport(textMateLanguage); -} - -/** - * Initialize TextMate. Call this early to preload the grammar. - */ -export async function initLuauTextMate(): Promise { - try { - await initTextMate(); - } catch (error) { - console.error('[TextMate] Failed to initialize:', error); - } -} - diff --git a/vite.config.ts b/vite.config.ts index eefad73..16bdbae 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -7,13 +7,12 @@ import path from 'path'; import { preloadDynamicChunks } from './vite/preload-chunks'; import { prerenderPlugin } from './vite/prerender'; import { inlineCss } from './vite/inline-css'; -import { compileGrammarPlugin } from './vite/compile-grammar'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // https://vite.dev/config/ export default defineConfig({ - plugins: [compileGrammarPlugin(), svelte(), tailwindcss(), preloadDynamicChunks(), prerenderPlugin(), inlineCss()], + plugins: [svelte(), tailwindcss(), preloadDynamicChunks(), prerenderPlugin(), inlineCss()], resolve: { alias: { '$lib': path.resolve(__dirname, './src/lib'), diff --git a/vite/compile-grammar.ts b/vite/compile-grammar.ts deleted file mode 100644 index 3040530..0000000 --- a/vite/compile-grammar.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Vite plugin to compile Oniguruma regex patterns from the Luau TextMate grammar - * to native JavaScript RegExp patterns at build time. - * - * This eliminates the need for oniguruma-to-es at runtime. - * - * Usage: - * import { compiledPatterns } from 'virtual:compiled-patterns'; - */ - -import type { Plugin } from 'vite'; -import { toRegExp } from 'oniguruma-to-es'; -import { readFileSync } from 'fs'; -import { resolve } from 'path'; - -const VIRTUAL_MODULE_ID = 'virtual:compiled-patterns'; -const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID; - -interface GrammarNode { - match?: string; - begin?: string; - end?: string; - while?: string; - patterns?: GrammarNode[]; - repository?: Record; - captures?: Record; - beginCaptures?: Record; - endCaptures?: Record; - whileCaptures?: Record; - [key: string]: unknown; -} - -/** - * Extract all unique regex patterns from the grammar. - * Also generates vscode-textmate's internal transformations for anchored patterns. - */ -function extractPatterns(obj: unknown, patterns = new Set()): Set { - if (!obj || typeof obj !== 'object') return patterns; - - if (Array.isArray(obj)) { - for (const item of obj) { - extractPatterns(item, patterns); - } - } else { - const node = obj as GrammarNode; - // Check for pattern properties - for (const key of ['match', 'begin', 'end', 'while'] as const) { - if (typeof node[key] === 'string') { - const pattern = node[key]; - patterns.add(pattern); - - // vscode-textmate transforms \A anchors in patterns when isFirstLine=false - // It replaces the 'A' in '\A' with \xFFFF, so \A becomes \￿ (backslash + U+FFFF) - // We need to also compile these transformed versions - if (pattern.includes('\\A')) { - // Replace \A with \￿ (backslash followed by U+FFFF) - patterns.add(pattern.replace(/\\A/g, '\\\uFFFF')); - } - } - } - // Recurse into nested objects - for (const value of Object.values(node)) { - extractPatterns(value, patterns); - } - } - - return patterns; -} - -/** - * Compile all patterns to JavaScript RegExp. - */ -function compilePatterns(patterns: string[]): Record { - const compiled: Record = {}; - - for (const pattern of patterns) { - try { - // Handle vscode-textmate's anchor transformation - // vscode-textmate replaces \A with \￿ (backslash + U+FFFF) when isFirstLine=false - // We need to convert it back to \A for oniguruma-to-es compilation - let patternToCompile = pattern; - if (pattern.includes('\\\uFFFF')) { - patternToCompile = pattern.replace(/\\\uFFFF/g, '\\A'); - } - - const regex = toRegExp(patternToCompile, { - global: true, - hasIndices: true, - rules: { - allowOrphanBackrefs: true, - asciiWordBoundaries: true, - captureGroup: true, - recursionLimit: 5, - singleline: true, - }, - target: 'ES2024', - }); - - // Store as [source, flags] tuple - compiled[pattern] = [regex.source, regex.flags]; - } catch { - // Store null for patterns that couldn't be compiled - compiled[pattern] = null; - } - } - - return compiled; -} - -/** - * Generate the virtual module code. - */ -function generateModule(compiledPatterns: Record): string { - const total = Object.keys(compiledPatterns).length; - const failed = Object.values(compiledPatterns).filter(v => v === null).length; - - return `/** - * Pre-compiled regex patterns for the Luau TextMate grammar. - * Generated by vite/compile-grammar.ts - * - * Total patterns: ${total} - * Successfully compiled: ${total - failed} - * Failed: ${failed} - */ - -export const compiledPatterns = ${JSON.stringify(compiledPatterns, null, 2)}; -`; -} - -export function compileGrammarPlugin(): Plugin { - let grammarPath: string; - let cachedModule: string | null = null; - - return { - name: 'compile-grammar', - - configResolved(config) { - grammarPath = resolve(config.root, 'src/lib/editor/Luau.tmLanguage.json'); - }, - - resolveId(id) { - if (id === VIRTUAL_MODULE_ID) { - return RESOLVED_VIRTUAL_MODULE_ID; - } - }, - - load(id) { - if (id === RESOLVED_VIRTUAL_MODULE_ID) { - // Return cached module if available (for HMR performance) - if (cachedModule) { - return cachedModule; - } - - // Read and parse grammar - const grammarJson = readFileSync(grammarPath, 'utf-8'); - const grammar = JSON.parse(grammarJson); - - // Extract and compile patterns - const patterns = [...extractPatterns(grammar)]; - const compiled = compilePatterns(patterns); - - // Generate and cache module - cachedModule = generateModule(compiled); - - console.log(`[compile-grammar] Compiled ${patterns.length} patterns`); - - return cachedModule; - } - }, - - handleHotUpdate({ file }) { - // Invalidate cache when grammar file changes - if (file.endsWith('Luau.tmLanguage.json')) { - cachedModule = null; - console.log('[compile-grammar] Grammar changed, will recompile patterns'); - } - }, - }; -} - diff --git a/vite/prerender.ts b/vite/prerender.ts index d4c1301..28eec7c 100644 --- a/vite/prerender.ts +++ b/vite/prerender.ts @@ -4,7 +4,6 @@ import tailwindcss from '@tailwindcss/vite'; import { rmSync } from 'fs'; import { fileURLToPath } from 'url'; import path from 'path'; -import { compileGrammarPlugin } from './compile-grammar'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const rootDir = path.resolve(__dirname, '..'); @@ -27,7 +26,7 @@ export function prerenderPlugin(): Plugin { // Build SSR version await build({ configFile: false, - plugins: [compileGrammarPlugin(), svelte(), tailwindcss()], + plugins: [svelte(), tailwindcss()], resolve: { alias: { '$lib': path.resolve(rootDir, './src/lib') }, },