Skip to content

YAML / JSON parsing, performance improvement for large files #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
halvarsson merged 2 commits into from
Nov 10, 2025
Merged

YAML / JSON parsing, performance improvement for large files #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
30ebbcc
improve performance for json parsing of large files
halvarsson Nov 10, 2025
3e43929
improve yaml parsing for large files
halvarsson Nov 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 15 additions & 37 deletions EditorContext/JSONEditorContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace psedit
{
public class JSONEditorContext : EditorContext
{
private Dictionary<Point, Color> tokens;
private List<ParseResult> parsedTokens;
private List<ErrorParseResult> parsedErrors = new List<ErrorParseResult>();
public JSONEditorContext(int TabWidth)
{
Expand Down Expand Up @@ -62,9 +62,9 @@ public Terminal.Gui.Color GetColor(JsonToken token)

return textColor;
}
public Dictionary<Point, Color> ParseJsonToken(string text, List<List<Rune>> Runes)
public List<ParseResult> ParseJsonToken(string text, List<List<Rune>> Runes)
{
List<ParseResult> resultList = new List<ParseResult>();
List<ParseResult> returnValue = new List<ParseResult>();
Dictionary<Point, Color> returnList = new Dictionary<Point, Color>();

JsonTextReader reader = new JsonTextReader(new StringReader(text));
Expand All @@ -90,7 +90,7 @@ public Dictionary<Point, Color> ParseJsonToken(string text, List<List<Rune>> Run
var endIndex = reader.LinePosition;
var color = GetColor(reader.TokenType);
var result = new ParseResult { StartIndex = startIndex, EndIndex = endIndex, Color = color, LineNumber = lineNumber };
resultList.Add(result);
returnValue.Add(result);
oldPos = endIndex;
}
catch (JsonReaderException ex)
Expand Down Expand Up @@ -121,34 +121,7 @@ public Dictionary<Point, Color> ParseJsonToken(string text, List<List<Rune>> Run
oldPos = endIndex;
}
}

for (int idxRow = 0; idxRow < Runes.Count; idxRow++)
{
var line = EditorExtensions.GetLine(Runes, idxRow);
var tokenCol = 1;
int lineRuneCount = line.Count;
var rowTokens = resultList.Where(p => (p.LineNumber == idxRow + 1));
for (int idxCol = 0; idxCol < lineRuneCount; idxCol++)
{

var rune = idxCol >= lineRuneCount ? ' ' : line[idxCol];
var cols = Rune.ColumnWidth(rune);

var match = rowTokens.Where(p => (tokenCol >= p.StartIndex && tokenCol <= p.EndIndex)).FirstOrDefault();
var color = Color.Green;
if (match != null)
{
color = match.Color;
}

tokenCol++;
var point = new Point(idxCol + 1, idxRow + 1);
returnList.Add(point, color);

}
}

return returnList;
return returnValue;
}
public override void ParseText(int height, int topRow, int left, int right, string text, List<List<Rune>> Runes)
{
Expand All @@ -164,7 +137,7 @@ public override void ParseText(int height, int topRow, int left, int right, stri
parsedErrors.Clear();
Errors.Clear();
ColumnErrors.Clear();
tokens = ParseJsonToken(text, Runes);
parsedTokens = ParseJsonToken(text, Runes);
}

Dictionary<Point, Terminal.Gui.Color> returnDict = new Dictionary<Point, Terminal.Gui.Color>();
Expand All @@ -174,6 +147,7 @@ public override void ParseText(int height, int topRow, int left, int right, stri
_lastParseRightColumn = right;
long count = 0;
var row = 0;

for (int idxRow = topRow; idxRow < Runes.Count; idxRow++)
{
if (row > bottom)
Expand All @@ -184,7 +158,7 @@ public override void ParseText(int height, int topRow, int left, int right, stri
int lineRuneCount = line.Count;
var col = left;
var tokenCol = 1 + left;

var rowTokens = parsedTokens.Where(p => (p.LineNumber == idxRow + 1));
var rowErrors = parsedErrors.Where(e => e.LineNumber == idxRow + 1);

for (int idxCol = left; idxCol < lineRuneCount; idxCol++)
Expand All @@ -199,14 +173,18 @@ public override void ParseText(int height, int topRow, int left, int right, stri
{
ColumnErrors.TryAdd(new Point(idxCol, idxRow), colError.ErrorMessage);
}

var jsonParseMatch = tokens[new Point(tokenCol, idxRow + 1)];
var jsonParseMatch = rowTokens.Where(p => (tokenCol >= p.StartIndex && tokenCol <= p.EndIndex)).FirstOrDefault();
var color = Color.Green;
if (jsonParseMatch != null)
{
color = jsonParseMatch.Color;
}
count++;

var rune = idxCol >= lineRuneCount ? ' ' : line[idxCol];
var cols = Rune.ColumnWidth(rune);
var point = new Point(idxCol, row);
returnDict.Add(point, jsonParseMatch);
returnDict.Add(point, color);
tokenCol++;

if (!EditorExtensions.SetCol(ref col, right, cols))
Expand Down
49 changes: 16 additions & 33 deletions EditorContext/YamlEditorContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@ namespace psedit
{
public class YamlEditorContext : EditorContext
{
private Dictionary<Point, Color> tokens;
private List<ParseResult> parsedTokens;
private List<ErrorParseResult> parsedErrors = new List<ErrorParseResult>();
public YamlEditorContext(int TabWidth)
{
_tabWidth = TabWidth;
CanFormat = true;
CanSyntaxHighlight = true;
}
public Terminal.Gui.Color GetColor(ParsingEvent token)
public Terminal.Gui.Color GetColor(ParsingEvent token)
{
var theme = ThemeService.Instance;
switch (token)
Expand Down Expand Up @@ -49,10 +49,9 @@ public Terminal.Gui.Color GetColor(ParsingEvent token)
return theme.GetColor("Foreground");
}
}
public Dictionary<Point, Color> ParseYamlToken(string text, List<List<Rune>> Runes)
public List<ParseResult> ParseYamlToken(string text, List<List<Rune>> Runes)
{
List<ParseResult> resultList = new List<ParseResult>();
Dictionary<Point, Color> returnList = new Dictionary<Point, Color>();
List<ParseResult> returnValue = new List<ParseResult>();
var reader = new StringReader(text);
var parser = new YamlDotNet.Core.Parser(reader);
int oldPos = 1;
Expand Down Expand Up @@ -80,11 +79,10 @@ public Dictionary<Point, Color> ParseYamlToken(string text, List<List<Rune>> Run
oldPos = 1;
}
var startIndex = parser.Current != null ? (int)parser.Current.Start.Column : oldPos + 1;
var endIndex = parser.Current != null ? (int)parser.Current.End.Column -1 : oldPos + 1;
var endIndex = parser.Current != null ? (int)parser.Current.End.Column - 1 : oldPos + 1;
var color = GetColor(token);
var result = new ParseResult { StartIndex = startIndex, EndIndex = endIndex, Color = color, LineNumber = lineNumber };
Debug.WriteLine($"YAML Token: {token} Line: {lineNumber} StartIndex: {startIndex} EndIndex: {endIndex} Color: {color}");
resultList.Add(result);
returnValue.Add(result);
oldPos = endIndex;
}
catch (YamlDotNet.Core.YamlException ex)
Expand Down Expand Up @@ -113,28 +111,7 @@ public Dictionary<Point, Color> ParseYamlToken(string text, List<List<Rune>> Run
oldPos = (int)endIndex;
}
}
for (int idxRow = 0; idxRow < Runes.Count; idxRow++)
{
var line = EditorExtensions.GetLine(Runes, idxRow);
var tokenCol = 1;
int lineRuneCount = line.Count;
var rowTokens = resultList.Where(p => (p.LineNumber == idxRow + 1));
for (int idxCol = 0; idxCol < lineRuneCount; idxCol++)
{
var rune = idxCol >= lineRuneCount ? ' ' : line[idxCol];
var cols = Rune.ColumnWidth(rune);
var match = rowTokens.Where(p => (tokenCol >= p.StartIndex && tokenCol <= p.EndIndex)).FirstOrDefault();
var color = Color.Green;
if (match != null)
{
color = match.Color;
}
tokenCol++;
var point = new Point(idxCol + 1, idxRow + 1);
returnList.Add(point, color);
}
}
return returnList;
return returnValue;
}
public override void ParseText(int height, int topRow, int left, int right, string text, List<List<Rune>> Runes)
{
Expand All @@ -147,7 +124,7 @@ public override void ParseText(int height, int topRow, int left, int right, stri
parsedErrors.Clear();
Errors.Clear();
ColumnErrors.Clear();
tokens = ParseYamlToken(text, Runes);
parsedTokens = ParseYamlToken(text, Runes);
}
Dictionary<Point, Terminal.Gui.Color> returnDict = new Dictionary<Point, Terminal.Gui.Color>();
int bottom = topRow + height;
Expand All @@ -166,6 +143,7 @@ public override void ParseText(int height, int topRow, int left, int right, stri
int lineRuneCount = line.Count;
var col = left;
var tokenCol = 1 + left;
var rowTokens = parsedTokens.Where(p => (p.LineNumber == idxRow + 1));
var rowErrors = parsedErrors.Where(e => e.LineNumber == idxRow + 1);
for (int idxCol = left; idxCol < lineRuneCount; idxCol++)
{
Expand All @@ -178,12 +156,17 @@ public override void ParseText(int height, int topRow, int left, int right, stri
{
ColumnErrors.TryAdd(new Point(idxCol, idxRow), colError.ErrorMessage);
}
var yamlParseMatch = tokens[new Point(tokenCol, idxRow + 1)];
var yamlParseMatch = rowTokens.Where(p => (tokenCol >= p.StartIndex && tokenCol <= p.EndIndex)).FirstOrDefault();
var color = Color.Green;
if (yamlParseMatch != null)
{
color = yamlParseMatch.Color;
}
count++;
var rune = idxCol >= lineRuneCount ? ' ' : line[idxCol];
var cols = Rune.ColumnWidth(rune);
var point = new Point(idxCol, row);
returnDict.Add(point, yamlParseMatch);
returnDict.Add(point, color);
tokenCol++;
if (!EditorExtensions.SetCol(ref col, right, cols))
{
Expand Down