Lezer parser

package.json

@@ -11,9 +11,13 @@
     "prebuild": "npm run build:wasm",
     "build": "vite build",
     "preview": "vite preview",
-    "check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json"
+    "check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json",
+    "build:luau-parser": "lezer-generator src/lib/codemirror-lang-luau/luau.grammar -o src/lib/codemirror-lang-luau/parser.js",
+    "test:luau-parser": "node scripts/test-luau-parser.mjs",
+    "test:luau-highlighting": "node scripts/test-luau-highlighting.mjs"
   },
   "devDependencies": {
+    "@lezer/generator": "^1.8.0",
     "@sveltejs/vite-plugin-svelte": "^6.2.1",
     "@tailwindcss/vite": "^4.1.18",
     "@tsconfig/svelte": "^5.0.6",

scripts/luau-corpus.luau

@@ -0,0 +1,67 @@
+-- regular comment
+--[=[
+long comment with [=[ nested text
+]=]
+
+export type Result<T> = { ok: boolean, value: T? }
+
+type Pair<T, U = string> = { first: T, second: U }
+
+type function Mapper<T, U>(value: T): U
+
+local function make<T>(value: T): T
+  return value
+end
+
+local function returnsNothing()
+  return
+end
+
+local function packer<T...>(...: T...): T...
+  return ...
+end
+
+local function returnsTuple(): (number, string, ...boolean)
+  return 1, "ok", true
+end
+
+-- highlighting corpus
+local number = 10
+local type = "identifier"
+local continue = 42
+
+local function makeFoo<T>(bar: number, ...: T): T
+  return bar
+end
+
+local made = makeFoo(1)
+
+type Vec2 = { read x: number, write y: string }
+export type Alias<T> = Vec2
+
+type Callback = (x: number, y: string) -> number
+
+local typed: typeof(number) = number
+local casted = number :: Vec2
+
+number += 5
+number //= 2
+number ..= "!"
+
+local msg = `value is {number}, next is {number + 1}`
+local empty = ``
+
+local long = [=[line one
+line two]=]
+
+for i = 1, 10 do
+  if i % 2 == 0 then
+    continue
+  end
+end
+
+for _, n in numbers do
+  sum += n
+end
+
+local t = { a = 1, [2] = 3 }

scripts/test-luau-highlighting.mjs

@@ -0,0 +1,120 @@
+import { readFileSync } from "node:fs";
+import assert from "node:assert/strict";
+import { highlightTree, tagHighlighter, tags as t } from "@lezer/highlight";
+import { luauLanguage } from "../src/lib/codemirror-lang-luau/index.js";
+
+const source = readFileSync(new URL("./luau-corpus.luau", import.meta.url), "utf8");
+const tree = luauLanguage.parser.parse(source);
+
+const testHighlighter = tagHighlighter([
+  { tag: t.function(t.variableName), class: "fn" },
+  { tag: t.definition(t.variableName), class: "var-def" },
+  { tag: t.definition(t.typeName), class: "type-def" },
+  { tag: t.variableName, class: "var" },
+  { tag: t.typeName, class: "type" },
+  { tag: t.controlKeyword, class: "kw" },
+  { tag: t.definitionKeyword, class: "kw" },
+  { tag: t.operatorKeyword, class: "kw" },
+  { tag: t.keyword, class: "kw" },
+  { tag: t.modifier, class: "modifier" },
+  { tag: t.operator, class: "op" },
+  { tag: t.definitionOperator, class: "op" },
+  { tag: t.compareOperator, class: "op" },
+  { tag: t.arithmeticOperator, class: "op" },
+  { tag: t.bitwiseOperator, class: "op" },
+  { tag: t.typeOperator, class: "op" },
+  { tag: t.string, class: "string" },
+  { tag: t.special(t.string), class: "string" },
+  { tag: t.number, class: "number" },
+  { tag: t.comment, class: "comment" },
+  { tag: t.punctuation, class: "punct" },
+  { tag: t.meta, class: "meta" },
+]);
+
+const classes = Array.from({ length: source.length }, () => new Set());
+
+highlightTree(tree, testHighlighter, (from, to, cls) => {
+  const names = cls.split(" ").filter(Boolean);
+  for (let i = from; i < to; i++) {
+    for (const name of names) {
+      classes[i].add(name);
+    }
+  }
+});
+
+function findAnchor(anchor) {
+  const start = source.indexOf(anchor);
+  assert.ok(start >= 0, `anchor not found: ${anchor}`);
+  return start;
+}
+
+function assertTokenClass({ anchor, token, className }) {
+  const start = findAnchor(anchor);
+  const tokenOffset = anchor.indexOf(token);
+  let from;
+  if (tokenOffset >= 0) {
+    from = start + tokenOffset;
+  } else {
+    const found = source.indexOf(token, start);
+    assert.ok(found >= 0, `token not found after anchor: ${token} in ${anchor}`);
+    from = found;
+  }
+  const to = from + token.length;
+  for (let i = from; i < to; i++) {
+    assert.ok(
+      classes[i].has(className),
+      `expected ${className} on '${token}' at ${from}, got ${Array.from(classes[i]).join(", ")}`
+    );
+  }
+}
+
+const expectations = [
+  { anchor: "local number = 10", token: "local", className: "kw" },
+  { anchor: "local number = 10", token: "number", className: "var" },
+  { anchor: "local number = 10", token: "=", className: "op" },
+  { anchor: "local type = \"identifier\"", token: "type", className: "var" },
+  { anchor: "local type = \"identifier\"", token: "identifier", className: "string" },
+  { anchor: "local continue = 42", token: "continue", className: "var" },
+  { anchor: "\n    continue\n", token: "continue", className: "kw" },
+  { anchor: "local function makeFoo", token: "function", className: "kw" },
+  { anchor: "local function makeFoo", token: "makeFoo", className: "fn" },
+  { anchor: "makeFoo<T>(bar: number, ...: T): T", token: "T", className: "type-def" },
+  { anchor: "makeFoo<T>(bar: number, ...: T): T", token: "bar", className: "var-def" },
+  { anchor: "makeFoo<T>(bar: number, ...: T): T", token: "number", className: "type" },
+  { anchor: "...: T): T", token: "T", className: "type" },
+  { anchor: "local made = makeFoo(1)", token: "makeFoo", className: "fn" },
+  { anchor: "type Vec2 =", token: "type", className: "kw" },
+  { anchor: "type Vec2 =", token: "Vec2", className: "type-def" },
+  { anchor: "export type Alias", token: "export", className: "kw" },
+  { anchor: "export type Alias", token: "type", className: "kw" },
+  { anchor: "export type Alias", token: "Alias", className: "type-def" },
+  { anchor: "export type Alias", token: "Vec2", className: "type" },
+  { anchor: "packer<T...>(...: T...): T...", token: "T", className: "type-def" },
+  { anchor: "...: T...): T...", token: "T", className: "type" },
+  { anchor: "type Callback = (x: number, y: string) -> number", token: "x", className: "var-def" },
+  { anchor: "number, y: string", token: "y", className: "var-def" },
+  { anchor: "type Callback = (x: number, y: string) -> number", token: "->", className: "op" },
+  { anchor: "returnsTuple(): (number, string, ...boolean)", token: "number", className: "type" },
+  { anchor: "read x: number", token: "read", className: "modifier" },
+  { anchor: "write y: string", token: "write", className: "modifier" },
+  { anchor: "local typed: typeof(number)", token: "typeof", className: "kw" },
+  { anchor: "local typed: typeof(number)", token: "number", className: "var" },
+  { anchor: "local casted = number :: Vec2", token: "::", className: "op" },
+  { anchor: "local casted = number :: Vec2", token: "Vec2", className: "type" },
+  { anchor: "number += 5", token: "+=", className: "op" },
+  { anchor: "number //= 2", token: "//=", className: "op" },
+  { anchor: "number ..= \"!\"", token: "..=", className: "op" },
+  { anchor: "\n  return\nend\n\nlocal function packer", token: "return", className: "kw" },
+  { anchor: "local msg = `value is", token: "`value is ", className: "string" },
+  { anchor: "local long = [=[line one", token: "[=[line one\nline two]=]", className: "string" },
+  { anchor: "-- regular comment", token: "-- regular comment", className: "comment" },
+  { anchor: "--[=[\nlong comment", token: "--[=[\nlong comment with [=[ nested text\n]=]", className: "comment" },
+  { anchor: "for _, n in numbers do", token: "_", className: "var-def" },
+  { anchor: "for _, n in numbers do", token: "n", className: "var-def" },
+];
+
+for (const expectation of expectations) {
+  assertTokenClass(expectation);
+}
+
+console.log(`Luau highlight test passed (${expectations.length} checks).`);

scripts/test-luau-parser.mjs

@@ -0,0 +1,11 @@
+import { readFileSync } from "node:fs";
+import assert from "node:assert/strict";
+import { parser } from "../src/lib/codemirror-lang-luau/parser.js";
+
+const source = readFileSync(new URL("./luau-corpus.luau", import.meta.url), "utf8");
+const tree = parser.parse(source);
+
+assert.ok(tree, "parser returned a tree");
+assert.equal(tree.topNode.type.name, "Chunk", "root node is Chunk");
+
+console.log(`Parsed Luau corpus: ${tree.topNode.type.name} (${tree.length} chars)`);

src/lib/codemirror-lang-luau/index.js

@@ -0,0 +1,78 @@
+import { LRLanguage, LanguageSupport } from "@codemirror/language";
+import { styleTags, tags as t } from "@lezer/highlight";
+import { parser } from "./parser.js";
+
+const luauHighlight = styleTags({
+  "do then end while for in repeat until if elseif else return break": t.controlKeyword,
+  "local function": t.definitionKeyword,
+  "and or not": t.operatorKeyword,
+  "read write": t.modifier,
+  "typeof": t.operatorKeyword,
+  "ContinueStatement/continue": t.controlKeyword,
+  "TypeDeclaration/type ExportTypeDeclaration/type TypeFunctionDeclaration/type ExportTypeFunctionDeclaration/type": t.definitionKeyword,
+  "ExportTypeDeclaration/export ExportTypeFunctionDeclaration/export": t.definitionKeyword,
+  "true false": t.bool,
+  "nil": t.null,
+  Number: t.number,
+  String: t.string,
+  StringContentDouble: t.string,
+  StringContentSingle: t.string,
+  Escape: t.string,
+  LongString: t.string,
+  InterpChunk: t.string,
+  InterpStart: t.string,
+  InterpEnd: t.string,
+  LineComment: t.lineComment,
+  LongComment: t.blockComment,
+  Identifier: t.variableName,
+  Name: t.variableName,
+  "Param/Binding/Name/Identifier": t.definition(t.variableName),
+  "ForNumericStatement/Binding/Name/Identifier": t.definition(t.variableName),
+  "ForGenericStatement/BindingList/Binding/Name/Identifier": t.definition(t.variableName),
+  TypeName: t.typeName,
+  "TypeName/Identifier TypeName/Name/Identifier": t.typeName,
+  "GenericTypePack/Name/Identifier": t.typeName,
+  "TypeDeclaration/Identifier ExportTypeDeclaration/Identifier TypeFunctionDeclaration/Identifier ExportTypeFunctionDeclaration/Identifier": t.definition(t.typeName),
+  "TypeDeclaration/Name/Identifier ExportTypeDeclaration/Name/Identifier TypeFunctionDeclaration/Name/Identifier ExportTypeFunctionDeclaration/Name/Identifier": t.definition(t.typeName),
+  "GenericTypeParam/Identifier GenericTypeParam/Name/Identifier": t.definition(t.typeName),
+  "GenericTypeParamWithDefault/Identifier GenericTypeParamWithDefault/Name/Identifier": t.definition(t.typeName),
+  "GenericTypeParam/GenericTypePack/Name/Identifier GenericTypeParamWithDefault/GenericTypePack/Name/Identifier": t.definition(t.typeName),
+  "FunctionName/Identifier FunctionName/Name/Identifier": t.function(t.variableName),
+  "LocalFunctionDeclaration/Identifier LocalFunctionDeclaration/Name/Identifier": t.function(t.variableName),
+  "CallExpression/PrimaryExpression/Name CallExpression/PrimaryExpression/Name/Identifier": t.function(t.variableName),
+  "CallExpression/FieldSuffix/Name CallExpression/FieldSuffix/Name/Identifier": t.function(t.variableName),
+  "CallSuffix/Name CallSuffix/Name/Identifier": t.function(t.variableName),
+  "BoundType/Identifier BoundType/Name/Identifier": t.definition(t.variableName),
+  Attribute: t.meta,
+  AssignOp: t.definitionOperator,
+  CompoundAssignOp: t.definitionOperator,
+  CompareOp: t.compareOperator,
+  ShiftOp: t.bitwiseOperator,
+  UnaryOp: t.operator,
+  "TypeAnnotation/TypeColon ReturnType/TypeColon VarArgAnnotation/TypeColon TableProp/TypeColon TableIndexer/TypeColon BoundType/TypeColon": t.typeOperator,
+  "TypeCast TypeArrow": t.typeOperator,
+  "..": t.arithmeticOperator,
+  ".": t.derefOperator,
+  ", ;": t.separator,
+  TypeColon: t.punctuation,
+  ParenL: t.paren,
+  ParenR: t.paren,
+  BracketL: t.squareBracket,
+  BracketR: t.squareBracket,
+  BraceL: t.brace,
+  BraceR: t.brace,
+  "InterpOpen InterpClose": t.special(t.brace),
+});
+
+export const luauLanguage = LRLanguage.define({
+  parser: parser.configure({
+    props: [luauHighlight],
+  }),
+  languageData: {
+    commentTokens: { line: "--", block: { open: "--[[", close: "]]" } },
+  },
+});
+
+export function luau() {
+  return new LanguageSupport(luauLanguage);
+}