Skip to content

Commit

Permalink
fix #2201: update instantiation expression parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@evanw
evanw committed Feb 9, 2023
1 parent 88e17d8 commit 64a6388
Show file tree
Hide file tree
Showing 3 changed files with 198 additions and 70 deletions.
13 changes: 13 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,19 @@

## Unreleased

* Change esbuild's parsing of TypeScript instantiation expressions to match TypeScript 4.8+ ([#2201](https://github.com/evanw/esbuild/issues/2201))

This release updates esbuild's implementation of instantiation expression erasure to match [microsoft/TypeScript#49353](https://github.com/microsoft/TypeScript/pull/49353). The new rules are as follows (copied from TypeScript's PR description):

> When a potential type argument list is followed by
>
> * a line break,
> * an `(` token,
> * a template literal string, or
> * any token except `<` or `>` that isn't the start of an expression,
>
> we consider that construct to be a type argument list. Otherwise we consider the construct to be a `<` relational expression followed by a `>` relational expression.
* Ignore `sideEffects: false` for imported CSS files ([#1370](https://github.com/evanw/esbuild/issues/1370), [#1458](https://github.com/evanw/esbuild/pull/1458), [#2905](https://github.com/evanw/esbuild/issues/2905))

This release ignores the `sideEffects` annotation in `package.json` for CSS files that are imported into JS files using esbuild's `css` loader. This means that these CSS files are no longer be tree-shaken.
Expand Down
202 changes: 151 additions & 51 deletions internal/js_parser/ts_parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -806,7 +806,7 @@ func (p *parser) trySkipTypeScriptTypeArgumentsWithBacktracking() bool {

if p.skipTypeScriptTypeArguments(false /* isInsideJSXElement */) {
// Check the token after the type argument list and backtrack if it's invalid
if !p.canFollowTypeArgumentsInExpression() {
if !p.tsCanFollowTypeArgumentsInExpression() {
p.lexer.Unexpected()
}
}
Expand Down Expand Up @@ -944,22 +944,14 @@ func (p *parser) isTSArrowFnJSX() (isTSArrowFn bool) {
return
}

func (p *parser) nextTokenIsOpenParenOrLessThanOrDot() (result bool) {
oldLexer := p.lexer
p.lexer.Next()

result = p.lexer.Token == js_lexer.TOpenParen ||
p.lexer.Token == js_lexer.TLessThan ||
p.lexer.Token == js_lexer.TDot

// Restore the lexer
p.lexer = oldLexer
return
}

// This function is taken from the official TypeScript compiler source code:
// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
func (p *parser) canFollowTypeArgumentsInExpression() bool {
//
// This function is pretty inefficient as written, and could be collapsed into
// a single switch statement. But that would make it harder to keep this in
// sync with the TypeScript compiler's source code, so we keep doing it the
// slow way.
func (p *parser) tsCanFollowTypeArgumentsInExpression() bool {
switch p.lexer.Token {
case
// These tokens can follow a type argument list in a call expression.
Expand All @@ -968,9 +960,81 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool {
js_lexer.TTemplateHead: // foo<T> `...${100}...`
return true

// Consider something a type argument list only if the following token can't start an expression.
// A type argument list followed by `<` never makes sense, and a type argument list followed
// by `>` is ambiguous with a (re-scanned) `>>` operator, so we disqualify both. Also, in
// this context, `+` and `-` are unary operators, not binary operators.
case js_lexer.TLessThan,
js_lexer.TGreaterThan,
js_lexer.TPlus,
js_lexer.TMinus,
// TypeScript always sees "TGreaterThan" instead of these tokens since
// their scanner works a little differently than our lexer. So since
// "TGreaterThan" is forbidden above, we also forbid these too.
js_lexer.TGreaterThanEquals,
js_lexer.TGreaterThanGreaterThan,
js_lexer.TGreaterThanGreaterThanEquals,
js_lexer.TGreaterThanGreaterThanGreaterThan,
js_lexer.TGreaterThanGreaterThanGreaterThanEquals:
return false
}

// We favor the type argument list interpretation when it is immediately followed by
// a line break, a binary operator, or something that can't start an expression.
return p.lexer.HasNewlineBefore || p.tsIsBinaryOperator() || !p.tsIsStartOfExpression()
}

// This function is taken from the official TypeScript compiler source code:
// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
func (p *parser) tsIsBinaryOperator() bool {
switch p.lexer.Token {
case js_lexer.TIn:
return p.allowIn

case
js_lexer.TQuestionQuestion,
js_lexer.TBarBar,
js_lexer.TAmpersandAmpersand,
js_lexer.TBar,
js_lexer.TCaret,
js_lexer.TAmpersand,
js_lexer.TEqualsEquals,
js_lexer.TExclamationEquals,
js_lexer.TEqualsEqualsEquals,
js_lexer.TExclamationEqualsEquals,
js_lexer.TLessThan,
js_lexer.TGreaterThan,
js_lexer.TLessThanEquals,
js_lexer.TGreaterThanEquals,
js_lexer.TInstanceof,
js_lexer.TLessThanLessThan,
js_lexer.TGreaterThanGreaterThan,
js_lexer.TGreaterThanGreaterThanGreaterThan,
js_lexer.TPlus,
js_lexer.TMinus,
js_lexer.TAsterisk,
js_lexer.TSlash,
js_lexer.TPercent,
js_lexer.TAsteriskAsterisk:
return true

case js_lexer.TIdentifier:
if p.lexer.IsContextualKeyword("as") || p.lexer.IsContextualKeyword("satisfies") {
return true
}
}

return false
}

// This function is taken from the official TypeScript compiler source code:
// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
func (p *parser) tsIsStartOfExpression() bool {
if p.tsIsStartOfLeftHandSideExpression() {
return true
}

switch p.lexer.Token {
case
// From "isStartOfExpression()"
js_lexer.TPlus,
js_lexer.TMinus,
js_lexer.TTilde,
Expand All @@ -981,8 +1045,35 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool {
js_lexer.TPlusPlus,
js_lexer.TMinusMinus,
js_lexer.TLessThan,
js_lexer.TPrivateIdentifier,
js_lexer.TAt:
return true

// From "isStartOfLeftHandSideExpression()"
default:
if p.lexer.Token == js_lexer.TIdentifier && (p.lexer.Identifier.String == "await" || p.lexer.Identifier.String == "yield") {
// Yield/await always starts an expression. Either it is an identifier (in which case
// it is definitely an expression). Or it's a keyword (either because we're in
// a generator or async function, or in strict mode (or both)) and it started a yield or await expression.
return true
}

// Error tolerance. If we see the start of some binary operator, we consider
// that the start of an expression. That way we'll parse out a missing identifier,
// give a good message about an identifier being missing, and then consume the
// rest of the binary expression.
if p.tsIsBinaryOperator() {
return true
}

return p.tsIsIdentifier()
}
}

// This function is taken from the official TypeScript compiler source code:
// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
func (p *parser) tsIsStartOfLeftHandSideExpression() bool {
switch p.lexer.Token {
case
js_lexer.TThis,
js_lexer.TSuper,
js_lexer.TNull,
Expand All @@ -991,53 +1082,62 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool {
js_lexer.TNumericLiteral,
js_lexer.TBigIntegerLiteral,
js_lexer.TStringLiteral,
js_lexer.TNoSubstitutionTemplateLiteral,
js_lexer.TTemplateHead,
js_lexer.TOpenParen,
js_lexer.TOpenBracket,
js_lexer.TOpenBrace,
js_lexer.TFunction,
js_lexer.TClass,
js_lexer.TNew,
js_lexer.TSlash,
js_lexer.TSlashEquals,
js_lexer.TIdentifier,
js_lexer.TIdentifier:
return true

// From "isBinaryOperator()"
js_lexer.TQuestionQuestion,
js_lexer.TBarBar,
js_lexer.TAmpersandAmpersand,
js_lexer.TBar,
js_lexer.TCaret,
js_lexer.TAmpersand,
js_lexer.TEqualsEquals,
js_lexer.TExclamationEquals,
js_lexer.TEqualsEqualsEquals,
js_lexer.TExclamationEqualsEquals,
js_lexer.TGreaterThan,
js_lexer.TLessThanEquals,
js_lexer.TGreaterThanEquals,
js_lexer.TInstanceof,
js_lexer.TLessThanLessThan,
js_lexer.TGreaterThanGreaterThan,
js_lexer.TGreaterThanGreaterThanGreaterThan,
js_lexer.TAsterisk,
js_lexer.TPercent,
js_lexer.TAsteriskAsterisk,
case js_lexer.TImport:
return p.tsLookAheadNextTokenIsOpenParenOrLessThanOrDot()

// TypeScript always sees "TGreaterThan" instead of these tokens since
// their scanner works a little differently than our lexer. So since
// "TGreaterThan" is forbidden above, we also forbid these too.
js_lexer.TGreaterThanGreaterThanEquals,
js_lexer.TGreaterThanGreaterThanGreaterThanEquals:
return false
default:
return p.tsIsIdentifier()
}
}

case js_lexer.TIn:
return !p.allowIn
// This function is taken from the official TypeScript compiler source code:
// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
func (p *parser) tsLookAheadNextTokenIsOpenParenOrLessThanOrDot() (result bool) {
oldLexer := p.lexer
p.lexer.Next()

case js_lexer.TImport:
return !p.nextTokenIsOpenParenOrLessThanOrDot()
result = p.lexer.Token == js_lexer.TOpenParen ||
p.lexer.Token == js_lexer.TLessThan ||
p.lexer.Token == js_lexer.TDot

// Restore the lexer
p.lexer = oldLexer
return
}

// This function is taken from the official TypeScript compiler source code:
// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
func (p *parser) tsIsIdentifier() bool {
if p.lexer.Token == js_lexer.TIdentifier {
// If we have a 'yield' keyword, and we're in the [yield] context, then 'yield' is
// considered a keyword and is not an identifier.
if p.fnOrArrowDataParse.yield != allowIdent && p.lexer.Identifier.String == "yield" {
return false
}

// If we have a 'await' keyword, and we're in the [Await] context, then 'await' is
// considered a keyword and is not an identifier.
if p.fnOrArrowDataParse.await != allowIdent && p.lexer.Identifier.String == "await" {
return false
}

default:
return true
}

return false
}

func (p *parser) skipTypeScriptInterfaceStmt(opts parseStmtOpts) {
Expand Down
53 changes: 34 additions & 19 deletions internal/js_parser/ts_parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2014,7 +2014,7 @@ func TestTSInstantiationExpression(t *testing.T) {
// Function call
expectPrintedTS(t, "const x1 = f<true>\n(true);", "const x1 = f(true);\n")
// Relational expression
expectPrintedTS(t, "const x1 = f<true>\ntrue;", "const x1 = f < true > true;\n")
expectPrintedTS(t, "const x1 = f<true>\ntrue;", "const x1 = f;\ntrue;\n")
// Instantiation expression
expectPrintedTS(t, "const x1 = f<true>;\n(true);", "const x1 = f;\ntrue;\n")

Expand All @@ -2034,8 +2034,6 @@ func TestTSInstantiationExpression(t *testing.T) {
expectPrintedTS(t, "type T21 = typeof Array<string>; f();", "f();\n")
expectPrintedTS(t, "type T22 = typeof Array<string, number>; f();", "f();\n")

// This behavior matches TypeScript 4.7.0 nightly (specifically "typescript@4.7.0-dev.20220421")
// after various fixes from Microsoft that landed after the TypeScript 4.7.0 beta
expectPrintedTS(t, "f<x>, g<y>;", "f, g;\n")
expectPrintedTS(t, "f<<T>() => T>;", "f;\n")
expectPrintedTS(t, "f.x<<T>() => T>;", "f.x;\n")
Expand All @@ -2058,11 +2056,14 @@ func TestTSInstantiationExpression(t *testing.T) {
expectPrintedTS(t, "{ f<x> }", "{\n f;\n}\n")
expectPrintedTS(t, "f<x> + g<y>;", "f < x > +g;\n")
expectPrintedTS(t, "f<x> - g<y>;", "f < x > -g;\n")
expectParseErrorTS(t, "f<x> * g<y>;", "<stdin>: ERROR: Unexpected \"*\"\n")
expectParseErrorTS(t, "f<x> == g<y>;", "<stdin>: ERROR: Unexpected \"==\"\n")
expectParseErrorTS(t, "f<x> ?? g<y>;", "<stdin>: ERROR: Unexpected \"??\"\n")
expectParseErrorTS(t, "f<x> in g<y>;", "<stdin>: ERROR: Unexpected \"in\"\n")
expectParseErrorTS(t, "f<x> instanceof g<y>;", "<stdin>: ERROR: Unexpected \"instanceof\"\n")
expectPrintedTS(t, "f<x> * g<y>;", "f * g;\n")
expectPrintedTS(t, "f<x> *= g<y>;", "f *= g;\n")
expectPrintedTS(t, "f<x> == g<y>;", "f == g;\n")
expectPrintedTS(t, "f<x> ?? g<y>;", "f ?? g;\n")
expectPrintedTS(t, "f<x> in g<y>;", "f in g;\n")
expectPrintedTS(t, "f<x> instanceof g<y>;", "f instanceof g;\n")
expectPrintedTS(t, "f<x> as g<y>;", "f;\n")
expectPrintedTS(t, "f<x> satisfies g<y>;", "f;\n")

expectParseErrorTS(t, "const a8 = f<number><number>;", "<stdin>: ERROR: Unexpected \";\"\n")
expectParseErrorTS(t, "const b1 = f?.<number>;", "<stdin>: ERROR: Expected \"(\" but found \";\"\n")
Expand All @@ -2082,20 +2083,26 @@ func TestTSInstantiationExpression(t *testing.T) {
expectParseErrorTSX(t, "type x = typeof y\n<number>\nz", "<stdin>: ERROR: Unexpected end of file before a closing \"number\" tag\n<stdin>: NOTE: The opening \"number\" tag is here:\n")

// See: https://github.com/microsoft/TypeScript/issues/48654
expectPrintedTS(t, "x<true>\ny", "x < true > y;\n")
expectPrintedTS(t, "x<true> y", "x < true > y;\n")
expectPrintedTS(t, "x<true>\ny", "x;\ny;\n")
expectPrintedTS(t, "x<true>\nif (y) {}", "x;\nif (y) {\n}\n")
expectPrintedTS(t, "x<true>\nimport 'y'", "x;\nimport \"y\";\n")
expectPrintedTS(t, "x<true>\nimport('y')", "x < true > import(\"y\");\n")
expectPrintedTS(t, "x<true>\nimport.meta", "x < true > import.meta;\n")
expectPrintedTS(t, "new x<number>\ny", "new x() < number > y;\n")
expectPrintedTS(t, "x<true>\nimport('y')", "x;\nimport(\"y\");\n")
expectPrintedTS(t, "x<true>\nimport.meta", "x;\nimport.meta;\n")
expectPrintedTS(t, "x<true> import('y')", "x < true > import(\"y\");\n")
expectPrintedTS(t, "x<true> import.meta", "x < true > import.meta;\n")
expectPrintedTS(t, "new x<number> y", "new x() < number > y;\n")
expectPrintedTS(t, "new x<number>\ny", "new x();\ny;\n")
expectPrintedTS(t, "new x<number>\nif (y) {}", "new x();\nif (y) {\n}\n")
expectPrintedTS(t, "new x<true>\nimport 'y'", "new x();\nimport \"y\";\n")
expectPrintedTS(t, "new x<true>\nimport('y')", "new x() < true > import(\"y\");\n")
expectPrintedTS(t, "new x<true>\nimport.meta", "new x() < true > import.meta;\n")
expectPrintedTS(t, "new x<true>\nimport('y')", "new x();\nimport(\"y\");\n")
expectPrintedTS(t, "new x<true>\nimport.meta", "new x();\nimport.meta;\n")
expectPrintedTS(t, "new x<true> import('y')", "new x() < true > import(\"y\");\n")
expectPrintedTS(t, "new x<true> import.meta", "new x() < true > import.meta;\n")

// See: https://github.com/microsoft/TypeScript/issues/48759
expectParseErrorTS(t, "x<true>\nimport<T>('y')", "<stdin>: ERROR: Expected \"(\" but found \"<\"\n")
expectParseErrorTS(t, "new x<true>\nimport<T>('y')", "<stdin>: ERROR: Expected \"(\" but found \"<\"\n")
expectParseErrorTS(t, "x<true>\nimport<T>('y')", "<stdin>: ERROR: Unexpected \"<\"\n")
expectParseErrorTS(t, "new x<true>\nimport<T>('y')", "<stdin>: ERROR: Unexpected \"<\"\n")

// See: https://github.com/evanw/esbuild/issues/2201
expectParseErrorTS(t, "return Array < ;", "<stdin>: ERROR: Unexpected \";\"\n")
Expand All @@ -2115,12 +2122,20 @@ func TestTSInstantiationExpression(t *testing.T) {
expectPrintedTS(t, "return Array < Array < number >> +1;", "return Array < Array < number >> 1;\n")
expectPrintedTS(t, "return Array < Array < number >> (1);", "return Array(1);\n")
expectPrintedTS(t, "return Array < Array < number > > (1);", "return Array(1);\n")
expectParseErrorTS(t, "return Array < number > in x;", "<stdin>: ERROR: Unexpected \"in\"\n")
expectParseErrorTS(t, "return Array < Array < number >> in x;", "<stdin>: ERROR: Unexpected \"in\"\n")
expectParseErrorTS(t, "return Array < Array < number > > in x;", "<stdin>: ERROR: Unexpected \">\"\n")
expectPrintedTS(t, "return Array < number > in x;", "return Array in x;\n")
expectPrintedTS(t, "return Array < Array < number >> in x;", "return Array in x;\n")
expectPrintedTS(t, "return Array < Array < number > > in x;", "return Array in x;\n")
expectPrintedTS(t, "for (var x = Array < number > in y) ;", "x = Array;\nfor (var x in y)\n ;\n")
expectPrintedTS(t, "for (var x = Array < Array < number >> in y) ;", "x = Array;\nfor (var x in y)\n ;\n")
expectPrintedTS(t, "for (var x = Array < Array < number > > in y) ;", "x = Array;\nfor (var x in y)\n ;\n")

// See: https://github.com/microsoft/TypeScript/pull/49353
expectPrintedTS(t, "F<{}> 0", "F < {} > 0;\n")
expectPrintedTS(t, "F<{}> class F<T> {}", "F < {} > class F {\n};\n")
expectPrintedTS(t, "f<{}> function f<T>() {}", "f < {} > function f() {\n};\n")
expectPrintedTS(t, "F<{}>\n0", "F;\n0;\n")
expectPrintedTS(t, "F<{}>\nclass F<T> {}", "F;\nclass F {\n}\n")
expectPrintedTS(t, "f<{}>\nfunction f<T>() {}", "f;\nfunction f() {\n}\n")
}

func TestTSExponentiation(t *testing.T) {
Expand Down

0 comments on commit 64a6388

Please sign in to comment.