Skip to content

Commit

Permalink
Use int literals instead of refs for Python and JavaScript
Browse files Browse the repository at this point in the history 
Update getMultiTokenAlternativeDescriptor test

fixes antlr#3703

Signed-off-by: Ivan Kochurkin <kvanttt@gmail.com>
Signed-off-by: Eric Vergnaud <eric.vergnaud@wanadoo.fr>
  • Loading branch information
@KvanTTT @ericvergnaud
KvanTTT authored and ericvergnaud committed Sep 9, 2022
1 parent ec4184d commit 44598d0
Show file tree
Hide file tree
Showing 15 changed files with 107 additions and 92 deletions.
11 changes: 8 additions & 3 deletions runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,11 @@ private static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor() {
private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() {
final int tokensCount = 64;

StringBuilder rule = new StringBuilder("t: ");
StringBuilder rule = new StringBuilder("r1: ");
StringBuilder tokens = new StringBuilder();
StringBuilder input = new StringBuilder();
StringBuilder output = new StringBuilder();

for (int i = 0; i < tokensCount; i++) {
String currentToken = "T" + i;
rule.append(currentToken);
Expand All @@ -170,17 +171,21 @@ private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() {
input.append(currentToken).append(" ");
output.append(currentToken);
}
String currentToken = "T" + tokensCount;
tokens.append(currentToken).append(": '").append(currentToken).append("';\n");
input.append(currentToken).append(" ");
output.append(currentToken);

String grammar = "grammar P;\n" +
"r: t+ EOF {<writeln(\"$text\")>};\n" +
"r: (r1 | T" + tokensCount + ")+ EOF {<writeln(\"$text\")>};\n" +
rule + "\n" +
tokens + "\n" +
"WS: [ ]+ -> skip;";

return new RuntimeTestDescriptor(
GrammarType.Parser,
"MultiTokenAlternative",
"https://github.com/antlr/antlr4/issues/3698",
"https://github.com/antlr/antlr4/issues/3698, https://github.com/antlr/antlr4/issues/3703",
input.toString(),
output + "\n",
"",
Expand Down
15 changes: 7 additions & 8 deletions tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,7 @@ ErrorHandler.Sync(this);
<if(choice.label)><labelref(choice.label)> = TokenStream.LT(1);<endif>
<preamble; separator="\n">
switch (TokenStream.LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand All @@ -569,7 +569,7 @@ LL1OptionalBlock(choice, alts, error) ::= <<
State = <choice.stateNumber>;
ErrorHandler.Sync(this);
switch (TokenStream.LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand Down Expand Up @@ -676,15 +676,15 @@ Sync(s) ::= "Sync(<s.expecting.name>);"
ThrowNoViableAlt(t) ::= "throw new NoViableAltException(this);"

TestSetInline(s) ::= <<
<s.bitsets:{bits | <if(rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
<s.bitsets:{bits | <if(rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
>>

// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x3f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
// produces smaller bytecode only when bits.tokens contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
<testShiftInRange({<offsetShift(s.varName, bits.shift)>})> && ((1L \<\< <offsetShift(s.varName, bits.shift)>) & <bits.calculated>L) != 0
%>
Expand All @@ -698,13 +698,12 @@ offsetShift(shiftAmount, offset) ::= <%
<if(!isZero.(offset))>(<shiftAmount> - <offset>)<else><shiftAmount><endif>
%>

// produces more efficient bytecode when bits.ttypes contains at most two items
bitsetInlineComparison(s, bits) ::= <%
<bits.ttypes:{ttype | <s.varName>==<tokenType.(ttype)>}; separator=" || ">
<bits.tokens:{t | <s.varName>==<tokenType.(t.name)>}; separator=" || ">
%>

cases(ttypes) ::= <<
<ttypes:{t | case <tokenType.(t)>:}; separator="\n">
cases(tokens) ::= <<
<tokens:{t | case <tokenType.(t.name)>:}; separator="\n">
>>

InvokeRule(r, argExprsChunks) ::= <<
Expand Down
14 changes: 6 additions & 8 deletions tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,7 @@ _errHandler->sync(this);
<! TODO: untested !><if (choice.label)>LL1AltBlock(choice, preamble, alts, error) <labelref(choice.label)> = _input->LT(1);<endif>
<preamble; separator="\n">
switch (_input->LA(1)) {
<choice.altLook, alts: {look, alt | <cases(ttypes = look)> {
<choice.altLook, alts: {look, alt | <cases(tokens = look)> {
<alt>
break;
\}
Expand All @@ -667,7 +667,7 @@ LL1OptionalBlock(choice, alts, error) ::= <<
setState(<choice.stateNumber>);
_errHandler->sync(this);
switch (_input->LA(1)) {
<choice.altLook, alts: {look, alt | <cases(ttypes = look)> {
<choice.altLook, alts: {look, alt | <cases(tokens = look)> {
<alt>
break;
\}
Expand Down Expand Up @@ -793,15 +793,14 @@ ThrowNoViableAlt(t) ::= "throw NoViableAltException(this);"

TestSetInlineHeader(s) ::= "<! Required but unused. !>"
TestSetInline(s) ::= <<
<s.bitsets: {bits | <if (rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
<s.bitsets: {bits | <if (rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
>>

// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount> & ~ 0x3fULL) == 0)
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <<
<testShiftInRange({<offsetShift(s.varName, bits.shift)>})> &&
((1ULL \<\< <offsetShift(s.varName, bits.shift)>) & <bits.calculated>) != 0
Expand All @@ -816,13 +815,12 @@ offsetShift(shiftAmount, offset, prefix = false) ::= <%
<if (!isZero.(offset))>(<if (prefix)><parser.name>::<endif><shiftAmount> - <offset>)<else><if (prefix)><parser.name>::<endif><shiftAmount><endif>
%>

// produces more efficient bytecode when bits.ttypes contains at most two items
bitsetInlineComparison(s, bits) ::= <%
<bits.ttypes: {ttype | <s.varName> == <parser.name>::<ttype>}; separator = "\n\n|| ">
<bits.tokens: {t | <s.varName> == <parser.name>::<t.name>}; separator = "\n\n|| ">
%>

cases(ttypes) ::= <<
<ttypes: {t | case <parser.name>::<t>:}; separator="\n">
cases(tokens) ::= <<
<tokens: {t | case <parser.name>::<t.name>:}; separator="\n">
>>

InvokeRuleHeader(r, argExprsChunks) ::= "InvokeRuleHeader"
Expand Down
14 changes: 6 additions & 8 deletions tool/resources/org/antlr/v4/tool/templates/codegen/Dart/Dart.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ errorHandler.sync(this);
<if(choice.label)><labelref(choice.label)> = tokenStream.LT(1);<endif>
<preamble; separator="\n">
switch (tokenStream.LA(1)!) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand All @@ -403,7 +403,7 @@ LL1OptionalBlock(choice, alts, error) ::= <<
state = <choice.stateNumber>;
errorHandler.sync(this);
switch (tokenStream.LA(1)!) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand Down Expand Up @@ -510,15 +510,14 @@ Sync(s) ::= "sync(<s.expecting.name>);"
ThrowNoViableAlt(t) ::= "throw NoViableAltException(this);"

TestSetInline(s) ::= <<
<s.bitsets:{bits | <if(rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
<s.bitsets:{bits | <if(rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
>>

// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x3f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
<testShiftInRange({<offsetShift(s.varName, bits.shift)>})> && ((1 \<\< <offsetShift(s.varName, bits.shift)>) & <bits.calculated>) != 0
%>
Expand All @@ -532,13 +531,12 @@ offsetShift(shiftAmount, offset) ::= <%
<if(!isZero.(offset))>(<shiftAmount> - <offset>)<else><shiftAmount><endif>
%>

// produces more efficient bytecode when bits.ttypes contains at most two items
bitsetInlineComparison(s, bits) ::= <%
<bits.ttypes:{ttype | <s.varName> == TOKEN_<ttype>}; separator=" || ">
<bits.tokens:{t | <s.varName> == TOKEN_<t.name>}; separator=" || ">
%>

cases(ttypes) ::= <<
<ttypes:{t | case TOKEN_<t>:}; separator="\n">
cases(tokens) ::= <<
<tokens:{t | case TOKEN_<t.name>:}; separator="\n">
>>

InvokeRule(r, argExprsChunks) ::=<<
Expand Down
10 changes: 4 additions & 6 deletions tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ p.GetErrorHandler().Sync(p)

switch p.GetTokenStream().LA(1) {
<if(choice.altLook && alts)>
<choice.altLook, alts:{look, alt | case <look:{l | <parser.name><l>}; separator=", ">:
<choice.altLook, alts:{look, alt | case <look:{l | <parser.name><l.name>}; separator=", ">:
<alt>}; separator="\n\n">


Expand All @@ -571,7 +571,7 @@ p.GetErrorHandler().Sync(p)

switch p.GetTokenStream().LA(1) {
<if(choice.altLook && alts)>
<choice.altLook, alts:{look, alt | case <look:{l | <parser.name><l>}; separator=", ">:
<choice.altLook, alts:{look, alt | case <look:{l | <parser.name><l.name>}; separator=", ">:
<alt>}; separator="\n\n">


Expand Down Expand Up @@ -725,15 +725,14 @@ Sync(s) ::= "Sync(<s.expecting.name>)"
ThrowNoViableAlt(t) ::= "panic(antlr.NewNoViableAltException(p, nil, nil, nil, nil, nil))"

TestSetInline(s) ::= <<
<s.bitsets:{bits | <if(rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
<s.bitsets:{bits | <if(rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
>>

// Javascript language spec - shift operators are 32 bits long max
testShiftInRange(shiftAmount) ::= <<
(int64(<shiftAmount>) & ^0x3f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> && ((int64(1) \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) != 0
%>
Expand All @@ -751,9 +750,8 @@ offsetShiftType(shiftAmount, offset) ::= <%
<if(!isZero.(offset))>(<parser.name><shiftAmount> - <offset>)<else><parser.name><shiftAmount><endif>
%>

// produces more efficient bytecode when bits.ttypes contains at most two items
bitsetInlineComparison(s, bits) ::= <%
<bits.ttypes:{ttype | <s.varName> == <parser.name><ttype>}; separator=" || ">
<bits.tokens:{t | <s.varName> == <parser.name><t.name>}; separator=" || ">
%>

InvokeRule(r, argExprsChunks) ::= <<
Expand Down
14 changes: 6 additions & 8 deletions tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,7 @@ _errHandler.sync(this);
<if(choice.label)><labelref(choice.label)> = _input.LT(1);<endif>
<preamble; separator="\n">
switch (_input.LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand All @@ -498,7 +498,7 @@ LL1OptionalBlock(choice, alts, error) ::= <<
setState(<choice.stateNumber>);
_errHandler.sync(this);
switch (_input.LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand Down Expand Up @@ -605,15 +605,14 @@ Sync(s) ::= "sync(<s.expecting.name>);"
ThrowNoViableAlt(t) ::= "throw new NoViableAltException(this);"

TestSetInline(s) ::= <<
<s.bitsets:{bits | <if(rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
<s.bitsets:{bits | <if(rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
>>

// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x3f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
<testShiftInRange({<offsetShift(s.varName, bits.shift)>})> && ((1L \<\< <offsetShift(s.varName, bits.shift)>) & <bits.calculated>L) != 0
%>
Expand All @@ -627,13 +626,12 @@ offsetShift(shiftAmount, offset) ::= <%
<if(!isZero.(offset))>(<shiftAmount> - <offset>)<else><shiftAmount><endif>
%>

// produces more efficient bytecode when bits.ttypes contains at most two items
bitsetInlineComparison(s, bits) ::= <%
<bits.ttypes:{ttype | <s.varName>==<ttype>}; separator=" || ">
<bits.tokens:{t | <s.varName>==<t.name>}; separator=" || ">
%>

cases(ttypes) ::= <<
<ttypes:{t | case <t>:}; separator="\n">
cases(tokens) ::= <<
<tokens:{t | case <t.name>:}; separator="\n">
>>

InvokeRule(r, argExprsChunks) ::= <<
Expand Down
14 changes: 6 additions & 8 deletions tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ this._errHandler.sync(this);
<if(choice.label)><labelref(choice.label)> = this._input.LT(1);<endif>
<preamble; separator="\n">
switch(this._input.LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand All @@ -335,7 +335,7 @@ LL1OptionalBlock(choice, alts, error) ::= <<
this.state = <choice.stateNumber>;
this._errHandler.sync(this);
switch (this._input.LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n">
default:
Expand Down Expand Up @@ -445,15 +445,14 @@ Sync(s) ::= "sync(<s.expecting.name>)"
ThrowNoViableAlt(t) ::= "throw new antlr4.error.NoViableAltException(this);"

TestSetInline(s) ::= <<
<s.bitsets:{bits | <if(rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
<s.bitsets:{bits | <if(rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
>>

// Javascript language spec - shift operators are 32 bits long max
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x1f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> && ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) !== 0)
%>
Expand All @@ -471,13 +470,12 @@ offsetShiftType(shiftAmount, offset) ::= <%
<if(!isZero.(offset))>(<parser.name>.<shiftAmount> - <offset>)<else><parser.name>.<shiftAmount><endif>
%>

// produces more efficient bytecode when bits.ttypes contains at most two items
bitsetInlineComparison(s, bits) ::= <%
<bits.ttypes:{ttype | <s.varName>===<parser.name>.<ttype>}; separator=" || ">
<bits.tokens:{t | <s.varName>===<t.type>}; separator=" || ">
%>

cases(ttypes) ::= <<
<ttypes:{t | case <parser.name>.<t>:}; separator="\n">
cases(tokens) ::= <<
<tokens:{t | case <t.type>:}; separator="\n">
>>

InvokeRule(r, argExprsChunks) ::= <<
Expand Down
14 changes: 6 additions & 8 deletions tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,7 @@ $this->errorHandler->sync($this);
<preamble; separator="\n">

switch ($this->input->LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n\n">

Expand All @@ -547,7 +547,7 @@ $this->setState(<choice.stateNumber>);
$this->errorHandler->sync($this);

switch ($this->input->LA(1)) {
<choice.altLook,alts:{look,alt| <cases(ttypes=look)>
<choice.altLook,alts:{look,alt| <cases(tokens=look)>
<alt>
break;}; separator="\n\n">

Expand Down Expand Up @@ -667,15 +667,14 @@ Sync(s) ::= "sync(<s.expecting.name>);"
ThrowNoViableAlt(t) ::= "throw new NoViableAltException($this);"

TestSetInline(s) ::= <<
<s.bitsets:{bits | <if(rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
<s.bitsets:{bits | <if(rest(rest(bits.tokens)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">
>>

// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x3f) === 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> && ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) !== 0
%>
Expand All @@ -692,13 +691,12 @@ offsetShiftConst(shiftAmount, offset) ::= <%
<if(!isZero.(offset))>(self::<shiftAmount> - <offset>)<else>self::<shiftAmount><endif>
%>

// produces more efficient bytecode when bits.ttypes contains at most two items
bitsetInlineComparison(s, bits) ::= <%
<bits.ttypes:{ttype | $<s.varName> === self::<ttype>}; separator=" || ">
<bits.tokens:{t | $<s.varName> === self::<t.name>}; separator=" || ">
%>

cases(ttypes) ::= <<
<ttypes:{t | case self::<t>:}; separator="\n">
cases(tokens) ::= <<
<tokens:{t | case self::<t.name>:}; separator="\n">
>>

InvokeRule(r, argExprsChunks) ::= <<
Expand Down
Loading

0 comments on commit 44598d0

Please sign in to comment.