From e01ed02b8f00f4b8c6e4787a4b962d7fcfd66a23 Mon Sep 17 00:00:00 2001 From: Kevin Barabash Date: Mon, 30 Oct 2023 22:40:13 -0400 Subject: [PATCH] Parse patterns and statements (#2) * Parse patterns and statements * start TypeAnnParser, reorganize the code into modules * experiment with createParserForwardedToRef in new JsonParser * cleanup ExprParser a bit * add another test --- src/Escalier.Data/Library.fs | 16 +- src/Escalier.Parser.Tests/Tests.fs | 32 ++ .../snapshots/Tests.ParseFuncDef.verified.txt | 14 + .../Tests.ParseIntersectionType.verified.txt | 17 + ...ParseUnionAndIntersectionType.verified.txt | 28 ++ .../Tests.ParseUnionType.verified.txt | 17 + src/Escalier.Parser/Escalier.Parser.fsproj | 1 + src/Escalier.Parser/ExprParser.fs | 351 +++++++++++++++++- src/Escalier.Parser/JsonParser.fs | 72 ++++ src/Escalier/Program.fs | 12 +- 10 files changed, 531 insertions(+), 29 deletions(-) create mode 100644 src/Escalier.Parser.Tests/snapshots/Tests.ParseFuncDef.verified.txt create mode 100644 src/Escalier.Parser.Tests/snapshots/Tests.ParseIntersectionType.verified.txt create mode 100644 src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionAndIntersectionType.verified.txt create mode 100644 src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionType.verified.txt create mode 100644 src/Escalier.Parser/JsonParser.fs diff --git a/src/Escalier.Data/Library.fs b/src/Escalier.Data/Library.fs index 9d3c0598..a116e66f 100644 --- a/src/Escalier.Data/Library.fs +++ b/src/Escalier.Data/Library.fs @@ -38,9 +38,9 @@ module Syntax = [] type PatternKind = | Identifier of span: Span * name: string * is_mut: bool - | Object of elems: list + | Object of elems: list | Tuple of elems: list - | Wildcard of span: Span + | Wildcard | Literal of span: Span * value: Literal | Is of span: Span * target: Pattern * id: string * is_mut: bool @@ -145,6 +145,7 @@ module Syntax = | Undefined | Unknown | Never + | Object type TypeParam = { span: Span @@ -171,9 +172,8 @@ module Syntax = type MatchTypeCase = { extends: TypeAnn; true_type: TypeAnn } - type Ident = Span * string - type TypeAnnKind = + // TODO: collapse these into a single `Literal` type ann kind | BooleanLiteral of value: bool | NumberLiteral of value: string | StringLiteral of value: string @@ -183,11 +183,11 @@ module Syntax = | Array of elem: TypeAnn | Union of types: list | Intersection of types: list - | TypeRef of name: string * type_args: list + | TypeRef of name: string * type_args: option> | Function of FunctionType - | KeyOf of target: TypeAnn + | Keyof of target: TypeAnn | Rest of target: TypeAnn - | TypeOf of target: Ident + | Typeof of target: Expr | Index of target: TypeAnn * index: TypeAnn | Condition of ConditionType | Match of MatchType @@ -198,7 +198,7 @@ module Syntax = type TypeAnn = { kind: TypeAnnKind span: Span - inferred_type: option ref } + mutable inferred_type: option } module Type = type TypeParam = diff --git a/src/Escalier.Parser.Tests/Tests.fs b/src/Escalier.Parser.Tests/Tests.fs index c627855e..18b0e9cd 100644 --- a/src/Escalier.Parser.Tests/Tests.fs +++ b/src/Escalier.Parser.Tests/Tests.fs @@ -85,3 +85,35 @@ let ParseCallThenIndexer () = let result = sprintf "input: %s\noutput: %A" src expr Verifier.Verify(result, settings).ToTask() |> Async.AwaitTask + +[] +let ParseFuncDef () = + let src = "fn (x, y) { x }" + let expr = run ExprParser.func src + let result = sprintf "input: %s\noutput: %A" src expr + + Verifier.Verify(result, settings).ToTask() |> Async.AwaitTask + +[] +let ParseUnionType () = + let src = "number | string | boolean" + let expr = run ExprParser.typeAnn src + let result = sprintf "input: %s\noutput: %A" src expr + + Verifier.Verify(result, settings).ToTask() |> Async.AwaitTask + +[] +let ParseIntersectionType () = + let src = "number & string & boolean" + let expr = run ExprParser.typeAnn src + let result = sprintf "input: %s\noutput: %A" src expr + + Verifier.Verify(result, settings).ToTask() |> Async.AwaitTask + +[] +let ParseUnionAndIntersectionType () = + let src = "A & B | C & D" + let expr = run ExprParser.typeAnn src + let result = sprintf "input: %s\noutput: %A" src expr + + Verifier.Verify(result, settings).ToTask() |> Async.AwaitTask diff --git a/src/Escalier.Parser.Tests/snapshots/Tests.ParseFuncDef.verified.txt b/src/Escalier.Parser.Tests/snapshots/Tests.ParseFuncDef.verified.txt new file mode 100644 index 00000000..88752f56 --- /dev/null +++ b/src/Escalier.Parser.Tests/snapshots/Tests.ParseFuncDef.verified.txt @@ -0,0 +1,14 @@ +input: fn (x, y) { x } +output: Success: { kind = + Function + (["x"; "y"], Block { span = { start = 0 + stop = 0 } + stmts = [{ span = { start = 12 + stop = 14 } + kind = Expr { kind = Identifer "x" + span = { start = 12 + stop = 14 } + inferred_type = None } }] }) + span = { start = 0 + stop = 15 } + inferred_type = None } \ No newline at end of file diff --git a/src/Escalier.Parser.Tests/snapshots/Tests.ParseIntersectionType.verified.txt b/src/Escalier.Parser.Tests/snapshots/Tests.ParseIntersectionType.verified.txt new file mode 100644 index 00000000..bc4cf974 --- /dev/null +++ b/src/Escalier.Parser.Tests/snapshots/Tests.ParseIntersectionType.verified.txt @@ -0,0 +1,17 @@ +input: number & string & boolean +output: Success: { kind = + Intersection + [{ kind = Keyword Number + span = { start = 0 + stop = 7 } + inferred_type = None }; { kind = Keyword String + span = { start = 9 + stop = 16 } + inferred_type = None }; + { kind = Keyword Boolean + span = { start = 18 + stop = 25 } + inferred_type = None }] + span = { start = 0 + stop = 25 } + inferred_type = None } \ No newline at end of file diff --git a/src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionAndIntersectionType.verified.txt b/src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionAndIntersectionType.verified.txt new file mode 100644 index 00000000..89428f7b --- /dev/null +++ b/src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionAndIntersectionType.verified.txt @@ -0,0 +1,28 @@ +input: A & B | C & D +output: Success: { kind = + Union + [{ kind = + Intersection [{ kind = TypeRef ("A", None) + span = { start = 0 + stop = 2 } + inferred_type = None }; { kind = TypeRef ("B", None) + span = { start = 4 + stop = 6 } + inferred_type = None }] + span = { start = 0 + stop = 6 } + inferred_type = None }; + { kind = + Intersection [{ kind = TypeRef ("C", None) + span = { start = 8 + stop = 10 } + inferred_type = None }; { kind = TypeRef ("D", None) + span = { start = 12 + stop = 13 } + inferred_type = None }] + span = { start = 8 + stop = 13 } + inferred_type = None }] + span = { start = 0 + stop = 13 } + inferred_type = None } \ No newline at end of file diff --git a/src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionType.verified.txt b/src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionType.verified.txt new file mode 100644 index 00000000..2d7d9ebd --- /dev/null +++ b/src/Escalier.Parser.Tests/snapshots/Tests.ParseUnionType.verified.txt @@ -0,0 +1,17 @@ +input: number | string | boolean +output: Success: { kind = + Union + [{ kind = Keyword Number + span = { start = 0 + stop = 7 } + inferred_type = None }; { kind = Keyword String + span = { start = 9 + stop = 16 } + inferred_type = None }; + { kind = Keyword Boolean + span = { start = 18 + stop = 25 } + inferred_type = None }] + span = { start = 0 + stop = 25 } + inferred_type = None } \ No newline at end of file diff --git a/src/Escalier.Parser/Escalier.Parser.fsproj b/src/Escalier.Parser/Escalier.Parser.fsproj index b1e16927..c10b7a19 100644 --- a/src/Escalier.Parser/Escalier.Parser.fsproj +++ b/src/Escalier.Parser/Escalier.Parser.fsproj @@ -6,6 +6,7 @@ + diff --git a/src/Escalier.Parser/ExprParser.fs b/src/Escalier.Parser/ExprParser.fs index f995cc0e..042df705 100644 --- a/src/Escalier.Parser/ExprParser.fs +++ b/src/Escalier.Parser/ExprParser.fs @@ -4,34 +4,38 @@ open FParsec open Escalier.Data.Syntax open System.Text +exception ParseError of string + module ExprParser = - let mergeSpans (x: Span) (y: Span) = { start = x.start; stop = y.stop } + let ws = spaces + let str_ws s = pstring s .>> ws - let opp = new OperatorPrecedenceParser, unit>() - let expr: Parser = opp.ExpressionParser + let mergeSpans (x: Span) (y: Span) = { start = x.start; stop = y.stop } - let ws = spaces + let expr, exprRef = createParserForwardedToRef () + let stmt, stmtRef = createParserForwardedToRef () + let pattern, patternRef = createParserForwardedToRef () + let typeAnn, typeAnnRef = createParserForwardedToRef () + let primaryType, primaryTypeRef = createParserForwardedToRef () - let str_ws s = pstring s .>> ws + let opp = new OperatorPrecedenceParser, unit>() - let number: Parser = + let number: Parser = pipe3 getPosition pfloat getPosition <| fun p1 nl p2 -> let start = p1.Index |> int let stop = p2.Index |> int - { kind = Literal(Literal.Number(nl |> string)) - span = { start = start; stop = stop } - inferred_type = None } + Literal.Number(nl |> string) - let identifier: Parser = + let ident = let isIdentifierFirstChar c = isLetter c || c = '_' let isIdentifierChar c = isLetter c || isDigit c || c = '_' - let _ident = - many1Satisfy2L isIdentifierFirstChar isIdentifierChar "identifier" .>> ws // skips trailing whitespace + many1Satisfy2L isIdentifierFirstChar isIdentifierChar "identifier" .>> ws // skips trailing whitespace - pipe3 getPosition _ident getPosition + let identExpr: Parser = + pipe3 getPosition ident getPosition <| fun p1 sl p2 -> let start = p1.Index |> int let stop = p2.Index |> int @@ -40,7 +44,7 @@ module ExprParser = span = { start = start; stop = stop } inferred_type = None } - let stringLiteral: Parser = + let stringLiteral: Parser = let normalCharSnippet = manySatisfy (fun c -> c <> '\\' && c <> '"') let unescape c = @@ -63,10 +67,7 @@ module ExprParser = let start = p1.Index |> int let stop = p2.Index |> int - { kind = ExprKind.Literal(Literal.String(sl)) - span = { start = start; stop = stop } - inferred_type = None } - + Literal.String(sl) let templateStringLiteral: Parser = fun stream -> @@ -117,7 +118,43 @@ module ExprParser = else Reply(Error, messageError "Expected '`'") - let atom = number <|> identifier <|> stringLiteral <|> templateStringLiteral + let funcParam = ident // .>> (str_ws ":" >>. ident) + + let func: Parser = + pipe5 + getPosition + (str_ws "fn") + (between (str_ws "(") (str_ws ")") (sepBy funcParam (str_ws ","))) + (between (str_ws "{") (str_ws "}") (many stmt)) + getPosition + <| fun p1 _ params_ stmts p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + let body = + BlockOrExpr.Block( + { span = { start = 0; stop = 0 } + stmts = stmts } + ) + + { kind = ExprKind.Function(params_, body) + span = { start = start; stop = stop } + inferred_type = None } + + let literal = number <|> stringLiteral + + let literalExpr: Parser = + pipe3 getPosition literal getPosition + <| fun p1 lit p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { kind = ExprKind.Literal(lit) + span = { start = start; stop = stop } + inferred_type = None } + + let atom = literalExpr <|> identExpr <|> templateStringLiteral + let term = (atom .>> ws) <|> between (str_ws "(") (str_ws ")") expr opp.TermParser <- term @@ -390,3 +427,279 @@ module ExprParser = inferred_type = None }) ) ) + + exprRef.Value <- opp.ExpressionParser + + let private exprStmt: Parser = + pipe3 getPosition expr getPosition + <| fun p1 e p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { kind = StmtKind.Expr(e) + span = { start = start; stop = stop } } + + let private returnStmt: Parser = + pipe3 getPosition ((str_ws "return") >>. opt (expr)) getPosition + <| fun p1 e p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { kind = StmtKind.Return(e) + span = { start = start; stop = stop } } + + // `let = ` + let private varDecl = + pipe4 + getPosition + (str_ws "let" >>. pattern) + (str_ws "=" >>. expr) + getPosition + <| fun p1 pat init p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + let decl: Decl = + { kind = VarDecl(pat, Some(init), None, false) + span = { start = 0; stop = 0 } } + + { kind = StmtKind.Decl(decl) + span = { start = start; stop = stop } } + + // TODO: parse type params + let private typeDecl = + pipe4 + getPosition + (str_ws "type" >>. ident) + (str_ws "=" >>. typeAnn) + getPosition + <| fun p1 id typeAnn p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + let decl: Decl = + { kind = TypeDecl(id, typeAnn, None) + span = { start = 0; stop = 0 } } + + { kind = StmtKind.Decl(decl) + span = { start = start; stop = stop } } + + stmtRef.Value <- varDecl <|> typeDecl <|> returnStmt <|> exprStmt + + let private identPattern = + pipe3 getPosition ident getPosition + <| fun p1 id p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { Pattern.kind = + PatternKind.Identifier( + name = id, + is_mut = false, + span = { start = start; stop = stop } + ) + span = { start = start; stop = stop } + inferred_type = None } + + let private literalPattern = + pipe3 getPosition literal getPosition + <| fun p1 lit p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { Pattern.kind = + PatternKind.Literal( + span = { start = start; stop = stop }, + value = lit + ) + span = { start = start; stop = stop } + inferred_type = None } + + let private tuplePattern = + pipe3 + getPosition + (between (str_ws "[") (str_ws "]") (sepBy pattern (str_ws ","))) + getPosition + <| fun p1 patterns p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { Pattern.kind = PatternKind.Tuple(patterns) + span = { start = start; stop = stop } + inferred_type = None } + + let private wildcardPattern = + pipe3 getPosition (str_ws "_") getPosition + <| fun p1 _ p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { Pattern.kind = PatternKind.Wildcard + span = { start = start; stop = stop } + inferred_type = None } + + let private objPatKeyValue = + pipe4 getPosition ident pattern getPosition + <| fun p1 id pat p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + let span = { start = start; stop = stop } + KeyValue(span = span, key = id, value = pat, init = None) + + let private objPatElem = objPatKeyValue + + let private objectPattern = + pipe3 + getPosition + (between (str_ws "{") (str_ws "}") (sepBy objPatElem (str_ws ","))) + getPosition + <| fun p1 objElems p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + let span = { start = start; stop = stop } + + { Pattern.kind = PatternKind.Object(objElems) + span = span + inferred_type = None } + + patternRef.Value <- + identPattern + <|> literalPattern + <|> wildcardPattern + <|> objectPattern + <|> tuplePattern + + let private keywordTypeAnn = + + let keyword = + (str_ws "object" |>> fun _ -> KeywordType.Object) + <|> (str_ws "never" |>> fun _ -> Never) + <|> (str_ws "unknown" |>> fun _ -> Unknown) + <|> (str_ws "boolean" |>> fun _ -> Boolean) + <|> (str_ws "number" |>> fun _ -> Number) + <|> (str_ws "string" |>> fun _ -> String) + <|> (str_ws "symbol" |>> fun _ -> Symbol) + <|> (str_ws "null" |>> fun _ -> Null) + <|> (str_ws "undefined" |>> fun _ -> Undefined) + + pipe3 getPosition keyword getPosition + <| fun p1 keyword p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { TypeAnn.kind = TypeAnnKind.Keyword(keyword) + span = { start = start; stop = stop } + inferred_type = None } + + let private tupleTypeAnn = + pipe3 + getPosition + (between (str_ws "[") (str_ws "]") (sepBy typeAnn (str_ws ","))) + getPosition + <| fun p1 typeAnns p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { TypeAnn.kind = TypeAnnKind.Tuple(typeAnns) + span = { start = start; stop = stop } + inferred_type = None } + + let private keyofTypeAnn = + pipe3 getPosition (str_ws "keyof" >>. typeAnn) getPosition + <| fun p1 target p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { TypeAnn.kind = TypeAnnKind.Keyof(target) + span = { start = start; stop = stop } + inferred_type = None } + + let private restTypeAnn = + pipe3 getPosition (str_ws "..." >>. typeAnn) getPosition + <| fun p1 target p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { TypeAnn.kind = TypeAnnKind.Rest(target) + span = { start = start; stop = stop } + inferred_type = None } + + let private typeofTypeAnn = + pipe3 getPosition (str_ws "typeof" >>. expr) getPosition + <| fun p1 target p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { TypeAnn.kind = TypeAnnKind.Typeof(target) + span = { start = start; stop = stop } + inferred_type = None } + + let private typeRef = + pipe4 + getPosition + ident + (opt (between (str_ws "<") (str_ws ">") (sepBy typeAnn (str_ws ",")))) + getPosition + <| fun p1 name typeArgs p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { TypeAnn.kind = TypeAnnKind.TypeRef(name, typeArgs) + span = { start = start; stop = stop } + inferred_type = None } + + + let intersectionOrPrimaryType: Parser = + pipe3 getPosition (sepBy1 primaryType (str_ws "&")) getPosition + <| fun p1 typeAnns p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + match typeAnns with + | [ typeAnn ] -> typeAnn + | _ -> + { TypeAnn.kind = TypeAnnKind.Intersection(typeAnns) + span = { start = start; stop = stop } + inferred_type = None } + + let unionOrIntersectionOrPrimaryType: Parser = + pipe3 + getPosition + (sepBy1 intersectionOrPrimaryType (str_ws "|")) + getPosition + <| fun p1 typeAnns p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + match typeAnns with + | [ typeAnn ] -> typeAnn + | _ -> + { TypeAnn.kind = TypeAnnKind.Union(typeAnns) + span = { start = start; stop = stop } + inferred_type = None } + + let arrayTypeAnn = + pipe3 getPosition (primaryType .>> (str_ws "[]")) getPosition + <| fun p1 elem p2 -> + let start = p1.Index |> int + let stop = p2.Index |> int + + { TypeAnn.kind = TypeAnnKind.Array(elem) + span = { start = start; stop = stop } + inferred_type = None } + + primaryTypeRef.Value <- + // TODO: parathesized types + keywordTypeAnn // PredefinedType + // <|> objectTypeAnn + // <|> arrayTypeAnn + <|> tupleTypeAnn + <|> typeofTypeAnn // TypeQuery + <|> keyofTypeAnn + <|> restTypeAnn + // <|> thisTypeAnn + // should come last since any identifier can be a type reference + <|> typeRef + + // TODO: handle function types + typeAnnRef.Value <- unionOrIntersectionOrPrimaryType diff --git a/src/Escalier.Parser/JsonParser.fs b/src/Escalier.Parser/JsonParser.fs new file mode 100644 index 00000000..b855e36c --- /dev/null +++ b/src/Escalier.Parser/JsonParser.fs @@ -0,0 +1,72 @@ +namespace Escalier.Parser + +open FParsec + +module Json = + type Json = + | JString of string + | JNumber of float + | JBool of bool + | JNull + | JList of Json list + | JObject of Map + + let jvalue, jvalueRef = createParserForwardedToRef () + + let jnull = stringReturn "null" JNull + let jtrue = stringReturn "true" (JBool true) + let jfalse = stringReturn "false" (JBool false) + let jnumber = pfloat |>> JNumber + + let str s = pstring s + + let stringLiteral = + let escape = + anyOf "\"\\/bfnrt" + |>> function + | 'b' -> "\b" + | 'f' -> "\u000C" + | 'n' -> "\n" + | 'r' -> "\r" + | 't' -> "\t" + | c -> string c // every other char is mapped to itself + + let unicodeEscape = + /// converts a hex char ([0-9a-fA-F]) to its integer number (0-15) + let hex2int c = (int c &&& 15) + (int c >>> 6) * 9 + + str "u" + >>. pipe4 hex hex hex hex (fun h3 h2 h1 h0 -> + (hex2int h3) * 4096 + + (hex2int h2) * 256 + + (hex2int h1) * 16 + + hex2int h0 + |> char + |> string) + + let escapedCharSnippet = str "\\" >>. (escape <|> unicodeEscape) + let normalCharSnippet = manySatisfy (fun c -> c <> '"' && c <> '\\') + + between + (str "\"") + (str "\"") + (stringsSepBy normalCharSnippet escapedCharSnippet) + + let jstring = stringLiteral |>> JString + + let ws = spaces + + let listBetweenStrings sOpen sClose pElement f = + between + (str sOpen) + (str sClose) + (ws >>. sepBy (pElement .>> ws) (str "," >>. ws) |>> f) + + let jlist = listBetweenStrings "[" "]" jvalue JList + + let keyValue = stringLiteral .>>. (ws >>. str ":" >>. ws >>. jvalue) + let jobject = listBetweenStrings "{" "}" keyValue (Map.ofList >> JObject) + + do + jvalueRef.Value <- + choice [ jobject; jlist; jstring; jnumber; jtrue; jfalse; jnull ] diff --git a/src/Escalier/Program.fs b/src/Escalier/Program.fs index f67cfdb8..1e2690d4 100644 --- a/src/Escalier/Program.fs +++ b/src/Escalier/Program.fs @@ -3,8 +3,6 @@ printfn "Hello from F#" open FParsec open Escalier.Parser -open Escalier.Data.Syntax -open System.Collections.Generic let str s = pstring s // let floatBetweenBrackets = (str "[") >>. pfloat .>> (str "]") @@ -103,3 +101,13 @@ test ExprParser.expr "`a ${`b ${c} d` e`" test ExprParser.expr "array[0]()" test ExprParser.expr "foo()[0]" + +test ExprParser.func "fn (x, y) { return x + y }" +test ExprParser.func "fn (x, y) { return }" +test ExprParser.stmt "let sum = x + y" + +test ExprParser.typeAnn "number | string | boolean" +test ExprParser.typeAnn "number & string & boolean" +// TODO: figure out how to parse this +test ExprParser.typeAnn "number[]" +test ExprParser.typeAnn "Foo"