Report LexerEmptyModeStackException and recover instead of throwing e…

…mpty stack exception, fixes antlr#2006 Fixed for Java, C#, Python2/3, JavaScript, Go runtimes Need to be fixed: C++, Swift, Dart, PHP
KvanTTT · Oct 28, 2021 · cb031fd · cb031fd
1 parent 6af4c77
commit cb031fd
Show file tree

Hide file tree

Showing 17 changed files with 270 additions and 109 deletions.
diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerErrorsDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerErrorsDescriptors.java
@@ -7,6 +7,7 @@
 package org.antlr.v4.test.runtime.descriptors;
 
 import org.antlr.v4.test.runtime.BaseLexerTestDescriptor;
+import org.antlr.v4.test.runtime.BaseParserTestDescriptor;
 import org.antlr.v4.test.runtime.CommentHasStringValue;
 
 public class LexerErrorsDescriptors {
@@ -249,4 +250,31 @@ public static class StringsEmbeddedInActions_2 extends StringsEmbeddedInActions
 		public String output = "[@0,6:5='<EOF>',<-1>,1:6]\n";
 		public String errors = "line 1:0 token recognition error at: '[\"foo]'\n";
 	}
+
+	// https://github.com/antlr/antlr4/issues/2006
+	public static class ReportEmptyModeStackError extends BaseLexerTestDescriptor {
+		public String input = "aabbcc";
+
+		/**
+		 [@0,0:1='aa',<1>,1:0]
+		 [@1,2:3='bb',<2>,1:2]
+		 [@2,4:5='cc',<3>,1:4]
+		 [@3,6:5='<EOF>',<-1>,1:6]
+		 */
+		@CommentHasStringValue
+		public String output;
+
+		/**
+		 lexer grammar T;
+		 A: 'aa';
+		 B: 'bb' -> popMode;
+		 C: 'cc';
+		 */
+		@CommentHasStringValue
+		public String grammar;
+
+		public String errors = "line 1:2 Unable to pop mode because modes stack is empty at: 'bb'\n";
+		public String startRule = "";
+		public String grammarName = "T";
+	}
 }
diff --git a/runtime/CSharp/src/Lexer.cs b/runtime/CSharp/src/Lexer.cs
@@ -246,11 +246,14 @@ public virtual int PopMode()
         {
             if (_modeStack.Count == 0)
             {
-                throw new InvalidOperationException();
+                NotifyListeners(new LexerEmptyModeStackException(this, _input, _tokenStartCharIndex));
+            }
+            else
+            {
+                int mode = _modeStack.Pop();
+                Mode(mode);
             }
 
-            int mode = _modeStack.Pop();
-            Mode(mode);
             return _mode;
         }
 
@@ -553,12 +556,13 @@ public virtual void Recover(LexerNoViableAltException e)
             }
         }
 
-        public virtual void NotifyListeners(LexerNoViableAltException e)
+        public virtual void NotifyListeners(LexerException e)
         {
-            string text = _input.GetText(Interval.Of(_tokenStartCharIndex, _input.Index));
-            string msg = "token recognition error at: '" + GetErrorDisplay(text) + "'";
+            int lastIndexOffset = e is LexerEmptyModeStackException ? 1 : 0;
+            String renderedText = GetErrorDisplay(_input.GetText(Interval.Of(_tokenStartCharIndex, _input.Index - lastIndexOffset)));
+            String errorMessage = e.GetErrorMessage(renderedText);
             IAntlrErrorListener<int> listener = ErrorListenerDispatch;
-            listener.SyntaxError(ErrorOutput, this, 0, _tokenStartLine, _tokenStartColumn, msg, e);
+            listener.SyntaxError(ErrorOutput, this, 0, _tokenStartLine, _tokenStartColumn, errorMessage, e);
         }
 
         public virtual string GetErrorDisplay(string s)

diff --git a/runtime/CSharp/src/LexerEmptyModeStackException.cs b/runtime/CSharp/src/LexerEmptyModeStackException.cs
@@ -0,0 +1,14 @@
+namespace Antlr4.Runtime
+{
+    public class LexerEmptyModeStackException : LexerException
+    {
+        public LexerEmptyModeStackException(Lexer lexer, ICharStream input, int startIndex) : base(lexer, input, startIndex)
+        {
+        }
+
+        public override string GetErrorMessage(string input)
+        {
+            return "Unable to pop mode because modes stack is empty at: '" + input + "'";
+        }
+    }
+}
diff --git a/runtime/CSharp/src/LexerException.cs b/runtime/CSharp/src/LexerException.cs
@@ -0,0 +1,15 @@
+namespace Antlr4.Runtime
+{
+    public abstract class LexerException : RecognitionException
+    {
+        /** Matching attempted at what input index? */
+        public readonly int StartIndex;
+
+        protected LexerException(Lexer lexer, ICharStream input, int startIndex) : base(lexer, input)
+        {
+            StartIndex = startIndex;
+        }
+
+        public abstract string GetErrorMessage(string input);
+    }
+}
diff --git a/runtime/CSharp/src/LexerNoViableAltException.cs b/runtime/CSharp/src/LexerNoViableAltException.cs
@@ -3,63 +3,39 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 using System.Globalization;
-using Antlr4.Runtime;
 using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Misc;
-using Antlr4.Runtime.Sharpen;
 
 namespace Antlr4.Runtime
 {
     [System.Serializable]
-    public class LexerNoViableAltException : RecognitionException
+    public class LexerNoViableAltException : LexerException
     {
         private const long serialVersionUID = -730999203913001726L;
 
-        /// <summary>Matching attempted at what input index?</summary>
-        private readonly int startIndex;
-
         /// <summary>Which configurations did we try at input.index() that couldn't match input.LA(1)?</summary>
         [Nullable]
         private readonly ATNConfigSet deadEndConfigs;
 
         public LexerNoViableAltException(Lexer lexer, ICharStream input, int startIndex, ATNConfigSet deadEndConfigs)
-            : base(lexer, input)
+            : base(lexer, input, startIndex)
         {
-            this.startIndex = startIndex;
             this.deadEndConfigs = deadEndConfigs;
         }
 
-        public virtual int StartIndex
-        {
-            get
-            {
-                return startIndex;
-            }
-        }
-
         [Nullable]
-        public virtual ATNConfigSet DeadEndConfigs
-        {
-            get
-            {
-                return deadEndConfigs;
-            }
-        }
+        public virtual ATNConfigSet DeadEndConfigs => deadEndConfigs;
 
-        public override IIntStream InputStream
-        {
-            get
-            {
-                return (ICharStream)base.InputStream;
-            }
-        }
+        public override IIntStream InputStream => (ICharStream)base.InputStream;
+
+        public override string GetErrorMessage(string input) => "token recognition error at: '" + input + "'";
 
         public override string ToString()
         {
             string symbol = string.Empty;
-            if (startIndex >= 0 && startIndex < ((ICharStream)InputStream).Size)
+            if (StartIndex >= 0 && StartIndex < ((ICharStream)InputStream).Size)
             {
-                symbol = ((ICharStream)InputStream).GetText(Interval.Of(startIndex, startIndex));
+                symbol = ((ICharStream)InputStream).GetText(Interval.Of(StartIndex, StartIndex));
                 symbol = Utils.EscapeWhitespace(symbol, false);
             }
             return string.Format(CultureInfo.CurrentCulture, "{0}('{1}')", typeof(Antlr4.Runtime.LexerNoViableAltException).Name, symbol);

diff --git a/runtime/Go/antlr/errors.go b/runtime/Go/antlr/errors.go
@@ -96,13 +96,24 @@ func (b *BaseRecognitionException) String() string {
 	return b.message
 }
 
+type LexerException interface {
+	GetErrorMessage(token string) string
+}
+
 type LexerNoViableAltException struct {
 	*BaseRecognitionException
 
-	startIndex     int
+	startIndex int
+
 	deadEndConfigs ATNConfigSet
 }
 
+type LexerEmptyModeStackException struct {
+	*BaseRecognitionException
+
+	startIndex int
+}
+
 func NewLexerNoViableAltException(lexer Lexer, input CharStream, startIndex int, deadEndConfigs ATNConfigSet) *LexerNoViableAltException {
 
 	l := new(LexerNoViableAltException)
@@ -123,6 +134,25 @@ func (l *LexerNoViableAltException) String() string {
 	return "LexerNoViableAltException" + symbol
 }
 
+func (l *LexerNoViableAltException) GetErrorMessage(token string) string {
+	return "token recognition error at: '" + token + "'"
+}
+
+func NewLexerEmptyModeStackException(lexer Lexer, input CharStream, startIndex int) *LexerEmptyModeStackException {
+
+	l := new(LexerEmptyModeStackException)
+
+	l.BaseRecognitionException = NewBaseRecognitionException("", lexer, input, nil)
+
+	l.startIndex = startIndex
+
+	return l
+}
+
+func (l *LexerEmptyModeStackException) GetErrorMessage(token string) string {
+	return "Unable to pop mode because modes stack is empty at: '" + token + "'"
+}
+
 type NoViableAltException struct {
 	*BaseRecognitionException
 

diff --git a/runtime/Go/antlr/lexer.go b/runtime/Go/antlr/lexer.go
@@ -165,7 +165,7 @@ func (b *BaseLexer) setTokenFactory(f TokenFactory) {
 func (b *BaseLexer) safeMatch() (ret int) {
 	defer func() {
 		if e := recover(); e != nil {
-			if re, ok := e.(RecognitionException); ok {
+			if re, ok := e.(LexerException); ok {
 				b.notifyListeners(re) // Report error
 				b.Recover(re)
 				ret = LexerSkip // default
@@ -264,13 +264,14 @@ func (b *BaseLexer) PushMode(m int) {
 
 func (b *BaseLexer) PopMode() int {
 	if len(b.modeStack) == 0 {
-		panic("Empty Stack")
-	}
-	if LexerATNSimulatorDebug {
-		fmt.Println("popMode back to " + fmt.Sprint(b.modeStack[0:len(b.modeStack)-1]))
+		b.notifyListeners(NewLexerEmptyModeStackException(b, b.input, b.TokenStartCharIndex))
+	} else {
+		if LexerATNSimulatorDebug {
+			fmt.Println("popMode back to " + fmt.Sprint(b.modeStack[0:len(b.modeStack)-1]))
+		}
+		i, _ := b.modeStack.Pop()
+		b.mode = i
 	}
-	i, _ := b.modeStack.Pop()
-	b.mode = i
 	return b.mode
 }
 
@@ -373,13 +374,21 @@ func (b *BaseLexer) GetAllTokens() []Token {
 	return tokens
 }
 
-func (b *BaseLexer) notifyListeners(e RecognitionException) {
+func (b *BaseLexer) notifyListeners(e LexerException) {
+	_, ok := e.(*LexerEmptyModeStackException)
+	var lastIndexOffset int
+	if ok {
+		lastIndexOffset = 1
+	} else {
+		lastIndexOffset = 0
+	}
+
 	start := b.TokenStartCharIndex
-	stop := b.input.Index()
-	text := b.input.GetTextFromInterval(NewInterval(start, stop))
-	msg := "token recognition error at: '" + text + "'"
+	stop := b.input.Index() - lastIndexOffset
+	tokenText := b.input.GetTextFromInterval(NewInterval(start, stop))
+	msg := e.GetErrorMessage(tokenText)
 	listener := b.GetErrorListenerDispatch()
-	listener.SyntaxError(b, nil, b.TokenStartLine, b.TokenStartColumn, msg, e)
+	listener.SyntaxError(b, nil, b.TokenStartLine, b.TokenStartColumn, msg, e.(RecognitionException))
 }
 
 func (b *BaseLexer) getErrorDisplayForChar(c rune) string {
@@ -405,7 +414,7 @@ func (b *BaseLexer) getCharErrorDisplay(c rune) string {
 // it all works out. You can instead use the rule invocation stack
 // to do sophisticated error recovery if you are in a fragment rule.
 // /
-func (b *BaseLexer) Recover(re RecognitionException) {
+func (b *BaseLexer) Recover(re LexerException) {
 	if b.input.LA(1) != TokenEOF {
 		if _, ok := re.(*LexerNoViableAltException); ok {
 			// Skip a char and try again

diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java
@@ -189,9 +189,12 @@ public void pushMode(int m) {
 	}
 
 	public int popMode() {
-		if ( _modeStack.isEmpty() ) throw new EmptyStackException();
-		if ( LexerATNSimulator.debug ) System.out.println("popMode back to "+ _modeStack.peek());
-		mode( _modeStack.pop() );
+		if ( _modeStack.isEmpty() ) {
+			notifyListeners(new LexerEmptyModeStackException(this, _input, _tokenStartCharIndex));
+		} else {
+			if (LexerATNSimulator.debug) System.out.println("popMode back to " + _modeStack.peek());
+			mode(_modeStack.pop());
+		}
 		return _mode;
 	}
 
@@ -356,12 +359,12 @@ public void recover(LexerNoViableAltException e) {
 		}
 	}
 
-	public void notifyListeners(LexerNoViableAltException e) {
-		String text = _input.getText(Interval.of(_tokenStartCharIndex, _input.index()));
-		String msg = "token recognition error at: '"+ getErrorDisplay(text) + "'";
-
+	public void notifyListeners(LexerException e) {
+		int lastIndexOffset = (e instanceof LexerEmptyModeStackException) ? 1 : 0;
+		String renderedText = getErrorDisplay(_input.getText(Interval.of(_tokenStartCharIndex, _input.index() - lastIndexOffset)));
+		String errorMessage = e.getErrorMessage(renderedText);
 		ANTLRErrorListener listener = getErrorListenerDispatch();
-		listener.syntaxError(this, null, _tokenStartLine, _tokenStartCharPositionInLine, msg, e);
+		listener.syntaxError(this, null, _tokenStartLine, _tokenStartCharPositionInLine, errorMessage, e);
 	}
 
 	public String getErrorDisplay(String s) {

diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerEmptyModeStackException.java b/runtime/Java/src/org/antlr/v4/runtime/LexerEmptyModeStackException.java
@@ -0,0 +1,12 @@
+package org.antlr.v4.runtime;
+
+public class LexerEmptyModeStackException extends LexerException {
+	public LexerEmptyModeStackException(Lexer lexer, IntStream input, int startIndex) {
+		super(lexer, input, startIndex);
+	}
+
+	@Override
+	public String getErrorMessage(String input) {
+		return "Unable to pop mode because modes stack is empty at: '" + input + "'";
+	}
+}
diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerException.java b/runtime/Java/src/org/antlr/v4/runtime/LexerException.java
@@ -0,0 +1,17 @@
+package org.antlr.v4.runtime;
+
+public abstract class LexerException extends RecognitionException {
+	/** Matching attempted at what input index? */
+	protected final int startIndex;
+
+	public LexerException(Lexer lexer, IntStream input, int startIndex) {
+		super(lexer, input, null);
+		this.startIndex = startIndex;
+	}
+
+	public int getStartIndex() {
+		return startIndex;
+	}
+
+	public abstract String getErrorMessage(String input);
+}