From 1b08e0858b2701cf337a144bd2d189cd424b0c9c Mon Sep 17 00:00:00 2001
From: ChristianStadelmann
 <46969952+ChristianStadelmann@users.noreply.github.com>
Date: Tue, 19 Jan 2021 18:09:23 +0100
Subject: [PATCH] Tokenizer: Fix line continuation after punctuation

Prior to this change, any line ending with [punctuation + '...'], for
example `||...`, would cause the tokenizer to fail.
---
 tokenize_code.m | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tokenize_code.m b/tokenize_code.m
index dd5d628..7055653 100644
--- a/tokenize_code.m
+++ b/tokenize_code.m
@@ -99,6 +99,14 @@
             % any other operator:
             else
                 symbol = skip(punctuation);
+                % ends with '...':
+                % The '...' has to be unskipped and handled here in order
+                % to not cause and error for line endings such as `+...` 
+                % or `&&...`.
+                if length(symbol) > 3 && strcmp(symbol(end-2:end), '...')
+                    pos = pos - 3;
+                    symbol = symbol(1:end-3);
+                end
                 % one operator:
                 if any(strcmp(symbol, operators))
                     add_token('punctuation', symbol);