diff --git a/source/src/BNFC/Backend/C/CFtoFlexC.hs b/source/src/BNFC/Backend/C/CFtoFlexC.hs index cd0aecd6..096c704a 100644 --- a/source/src/BNFC/Backend/C/CFtoFlexC.hs +++ b/source/src/BNFC/Backend/C/CFtoFlexC.hs @@ -139,7 +139,7 @@ cMacros cf = unlines , "DIGIT [0-9]" , "IDENT [a-zA-Z0-9'_]" , unwords $ concat - [ [ "%START YYINITIAL CHAR CHARESC CHAREND STRING ESCAPED" ] + [ [ "%START CHAR CHARESC CHAREND STRING ESCAPED" ] , take (numberOfBlockCommentForms cf) commentStates ] , "" @@ -150,7 +150,7 @@ lexSymbols :: KeywordEnv -> String lexSymbols ss = concatMap transSym ss where transSym (s,r) = - "\"" ++ s' ++ "\" \t return " ++ r ++ ";\n" + "\"" ++ s' ++ "\" \t return " ++ r ++ ";\n" where s' = escapeChars s @@ -162,11 +162,11 @@ restOfFlex cf env = unlines $ concat , userDefTokens , ifC catString $ lexStrings "yylval" "_STRING_" "_ERROR_" , ifC catChar $ lexChars "yylval" "_CHAR_" - , ifC catDouble [ "{DIGIT}+\".\"{DIGIT}+(\"e\"(\\-)?{DIGIT}+)? \t yylval._double = atof(yytext); return _DOUBLE_;" ] - , ifC catInteger [ "{DIGIT}+ \t yylval._int = atoi(yytext); return _INTEGER_;" ] - , ifC catIdent [ "{LETTER}{IDENT}* \t yylval._string = strdup(yytext); return _IDENT_;" ] - , [ "[ \\t\\r\\n\\f] \t /* ignore white space. */;" - , ". \t return _ERROR_;" + , ifC catDouble [ "{DIGIT}+\".\"{DIGIT}+(\"e\"(\\-)?{DIGIT}+)? \t yylval._double = atof(yytext); return _DOUBLE_;" ] + , ifC catInteger [ "{DIGIT}+ \t yylval._int = atoi(yytext); return _INTEGER_;" ] + , ifC catIdent [ "{LETTER}{IDENT}* \t yylval._string = strdup(yytext); return _IDENT_;" ] + , [ "[ \\t\\r\\n\\f] \t /* ignore white space. */;" + , ". \t return _ERROR_;" , "" , "%% /* Initialization code. */" , "" @@ -176,7 +176,7 @@ restOfFlex cf env = unlines $ concat where ifC cat s = if isUsedCat cf (TokenCat cat) then s else [] userDefTokens = - [ "" ++ printRegFlex exp ++ + [ "" ++ printRegFlex exp ++ " \t yylval._string = strdup(yytext); return " ++ sName name ++ ";" | (name, exp) <- tokenPragmas cf ] @@ -190,16 +190,15 @@ restOfFlex cf env = unlines $ concat " yylloc.first_column = 1;", " yylloc.last_line = 1;", " yylloc.last_column = 1;", - " BEGIN YYINITIAL;", "}" ] -- | Lexing of strings, converting escaped characters. lexStrings :: String -> String -> String -> [String] lexStrings yylval stringToken errorToken = - [ "\"\\\"\" \t LITERAL_BUFFER_CREATE(); BEGIN STRING;" + [ "\"\\\"\" \t LITERAL_BUFFER_CREATE(); BEGIN STRING;" , "\\\\ \t BEGIN ESCAPED;" - , "\\\" \t " ++ yylval ++ "._string = LITERAL_BUFFER_HARVEST(); BEGIN YYINITIAL; return " ++ stringToken ++ ";" + , "\\\" \t " ++ yylval ++ "._string = LITERAL_BUFFER_HARVEST(); BEGIN INITIAL; return " ++ stringToken ++ ";" , ". \t LITERAL_BUFFER_APPEND_CHAR(yytext[0]);" , "n \t LITERAL_BUFFER_APPEND_CHAR('\\n'); BEGIN STRING;" , "\\\" \t LITERAL_BUFFER_APPEND_CHAR('\"'); BEGIN STRING;" @@ -212,13 +211,13 @@ lexStrings yylval stringToken errorToken = -- | Lexing of characters, converting escaped characters. lexChars :: String -> String -> [String] lexChars yylval charToken = - [ "\"'\" \tBEGIN CHAR;" + [ "\"'\" \tBEGIN CHAR;" , "\\\\ \t BEGIN CHARESC;" , "[^'] \t BEGIN CHAREND; " ++ yylval ++ "._char = yytext[0]; return " ++ charToken ++ ";" , "n \t BEGIN CHAREND; " ++ yylval ++ "._char = '\\n'; return " ++ charToken ++ ";" , "t \t BEGIN CHAREND; " ++ yylval ++ "._char = '\\t'; return " ++ charToken ++ ";" , ". \t BEGIN CHAREND; " ++ yylval ++ "._char = yytext[0]; return " ++ charToken ++ ";" - , "\"'\" \t BEGIN YYINITIAL;" + , "\"'\" \t BEGIN INITIAL;" ] -- --------------------------------------------------------------------------- @@ -233,9 +232,9 @@ lexChars yylval charToken = -- delimiters. -- -- >>> lexComments (Just "myns.") ([("{-","-}")],["--"]) --- "--"[^\n]* /* skip */; /* BNFC: comment "--" */ --- "{-" BEGIN COMMENT; /* BNFC: block comment "{-" "-}" */ --- "-}" BEGIN YYINITIAL; +-- "--"[^\n]* /* skip */; /* BNFC: comment "--" */ +-- "{-" BEGIN COMMENT; /* BNFC: block comment "{-" "-}" */ +-- "-}" BEGIN INITIAL; -- . /* skip */; -- [\n] /* skip */; lexComments :: Maybe String -> ([(String, String)], [String]) -> Doc @@ -254,13 +253,13 @@ commentStates = map ("COMMENT" ++) $ "" : map show [1..] -- comment. -- -- >>> lexSingleComment "--" --- "--"[^\n]* /* skip */; /* BNFC: comment "--" */ +-- "--"[^\n]* /* skip */; /* BNFC: comment "--" */ -- -- >>> lexSingleComment "\"" --- "\""[^\n]* /* skip */; /* BNFC: comment "\"" */ +-- "\""[^\n]* /* skip */; /* BNFC: comment "\"" */ lexSingleComment :: String -> Doc lexSingleComment c = - "" <> cstring c <> "[^\\n]*" + "" <> cstring c <> "[^\\n]*" <+> "/* skip */;" <+> unless (containsCCommentMarker c) ("/* BNFC: comment" <+> cstring c <+> "*/") @@ -276,22 +275,22 @@ containsCCommentMarker s = "/*" `isInfixOf` s || "*/" `isInfixOf` s -- with another. However this seems rare. -- -- >>> lexMultiComment ("{-", "-}") "COMMENT" --- "{-" BEGIN COMMENT; /* BNFC: block comment "{-" "-}" */ --- "-}" BEGIN YYINITIAL; +-- "{-" BEGIN COMMENT; /* BNFC: block comment "{-" "-}" */ +-- "-}" BEGIN INITIAL; -- . /* skip */; -- [\n] /* skip */; -- -- >>> lexMultiComment ("\"'", "'\"") "COMMENT" --- "\"'" BEGIN COMMENT; /* BNFC: block comment "\"'" "'\"" */ --- "'\"" BEGIN YYINITIAL; +-- "\"'" BEGIN COMMENT; /* BNFC: block comment "\"'" "'\"" */ +-- "'\"" BEGIN INITIAL; -- . /* skip */; -- [\n] /* skip */; lexMultiComment :: (String, String) -> String -> Doc lexMultiComment (b,e) comment = vcat - [ "" <> cstring b <+> "BEGIN" <+> text comment <> ";" + [ "" <> cstring b <+> "BEGIN" <+> text comment <> ";" <+> unless (containsCCommentMarker b || containsCCommentMarker e) ("/* BNFC: block comment" <+> cstring b <+> cstring e <+> "*/") - , commentTag <> cstring e <+> "BEGIN YYINITIAL;" + , commentTag <> cstring e <+> "BEGIN INITIAL;" , commentTag <> ". /* skip */;" , commentTag <> "[\\n] /* skip */;" ] diff --git a/source/src/BNFC/Backend/CPP/NoSTL/CFtoFlex.hs b/source/src/BNFC/Backend/CPP/NoSTL/CFtoFlex.hs index d9cbcb11..9818e7b1 100644 --- a/source/src/BNFC/Backend/CPP/NoSTL/CFtoFlex.hs +++ b/source/src/BNFC/Backend/CPP/NoSTL/CFtoFlex.hs @@ -62,7 +62,7 @@ lexSymbols :: SymEnv -> String lexSymbols ss = concatMap transSym ss where transSym (s,r) = - "\"" ++ s' ++ "\" \t return " ++ r ++ ";\n" + "\"" ++ s' ++ "\" \t return " ++ r ++ ";\n" where s' = escapeChars s @@ -74,12 +74,12 @@ restOfFlex inPackage cf env = unlines $ concat , userDefTokens , ifC catString $ lexStrings (ns ++ "yylval") (nsDefine inPackage "_STRING_") (nsDefine inPackage "_ERROR_") , ifC catChar $ lexChars (ns ++ "yylval") (nsDefine inPackage "_CHAR_") - , ifC catDouble [ "{DIGIT}+\".\"{DIGIT}+(\"e\"(\\-)?{DIGIT}+)? \t " ++ ns ++ "yylval._double = atof(yytext); return " ++ nsDefine inPackage "_DOUBLE_" ++ ";" ] - , ifC catInteger [ "{DIGIT}+ \t " ++ ns ++ "yylval._int = atoi(yytext); return " ++ nsDefine inPackage "_INTEGER_" ++ ";" ] - , ifC catIdent [ "{LETTER}{IDENT}* \t " ++ ns ++ "yylval._string = strdup(yytext); return " ++ nsDefine inPackage "_IDENT_" ++ ";" ] + , ifC catDouble [ "{DIGIT}+\".\"{DIGIT}+(\"e\"(\\-)?{DIGIT}+)? \t " ++ ns ++ "yylval._double = atof(yytext); return " ++ nsDefine inPackage "_DOUBLE_" ++ ";" ] + , ifC catInteger [ "{DIGIT}+ \t " ++ ns ++ "yylval._int = atoi(yytext); return " ++ nsDefine inPackage "_INTEGER_" ++ ";" ] + , ifC catIdent [ "{LETTER}{IDENT}* \t " ++ ns ++ "yylval._string = strdup(yytext); return " ++ nsDefine inPackage "_IDENT_" ++ ";" ] , [ "\\n ++" ++ ns ++ "yy_mylinenumber ;" - , "[ \\t\\r\\n\\f] \t /* ignore white space. */;" - , ". \t return " ++ nsDefine inPackage "_ERROR_" ++ ";" + , "[ \\t\\r\\n\\f] \t /* ignore white space. */;" + , ". \t return " ++ nsDefine inPackage "_ERROR_" ++ ";" , "%%" ] , footer @@ -88,13 +88,13 @@ restOfFlex inPackage cf env = unlines $ concat ifC cat s = if isUsedCat cf (TokenCat cat) then s else [] ns = nsString inPackage userDefTokens = - [ "" ++ printRegFlex exp ++ + [ "" ++ printRegFlex exp ++ " \t " ++ ns ++ "yylval._string = strdup(yytext); return " ++ sName name ++ ";" | (name, exp) <- tokenPragmas cf ] where sName n = fromMaybe n $ Map.lookup (Tokentype n) env footer = - [ "void " ++ ns ++ "initialize_lexer(FILE *inp) { yyrestart(inp); BEGIN YYINITIAL; }" + [ "void " ++ ns ++ "initialize_lexer(FILE *inp) { yyrestart(inp); }" , "int yywrap(void) { return 1; }" ] @@ -111,9 +111,9 @@ restOfFlex inPackage cf env = unlines $ concat -- delimiters. -- -- >>> lexComments (Just "myns.") ([("{-","-}")],["--"]) --- "--"[^\n]* ; // BNFC: comment "--"; --- "{-" BEGIN COMMENT; // BNFC: block comment "{-" "-}"; --- "-}" BEGIN YYINITIAL; +-- "--"[^\n]* ; // BNFC: comment "--"; +-- "{-" BEGIN COMMENT; // BNFC: block comment "{-" "-}"; +-- "-}" BEGIN INITIAL; -- . /* skip */; -- [\n] ++myns.yy_mylinenumber; lexComments :: Maybe String -> ([(String, String)], [String]) -> Doc @@ -128,38 +128,19 @@ lexComments ns (m,s) = vcat $ concat -- comment. -- -- >>> lexSingleComment (Just "mypackage.") "--" --- "--"[^\n]* ; // BNFC: comment "--"; +-- "--"[^\n]* ; // BNFC: comment "--"; -- -- >>> lexSingleComment Nothing "--" --- "--"[^\n]* ; // BNFC: comment "--"; +-- "--"[^\n]* ; // BNFC: comment "--"; -- -- >>> lexSingleComment Nothing "\"" --- "\""[^\n]* ; // BNFC: comment "\""; +-- "\""[^\n]* ; // BNFC: comment "\""; lexSingleComment :: Maybe String -> String -> Doc lexSingleComment _ c = - "" <> cstring c <> "[^\\n]*" + "" <> cstring c <> "[^\\n]*" <+> ";" <+> "// BNFC: comment" <+> cstring c <> ";" --- -- | Create a lexer rule for single-line comments. --- -- The first argument is -- an optional c++ namespace --- -- The second argument is the delimiter that marks the beginning of the --- -- comment. --- -- --- -- >>> lexSingleComment (Just "mypackage.") "--" --- -- "--"[^\n]*\n ++mypackage.yy_mylinenumber; // BNFC: comment "--"; --- -- --- -- >>> lexSingleComment Nothing "--" --- -- "--"[^\n]*\n ++yy_mylinenumber; // BNFC: comment "--"; --- -- --- -- >>> lexSingleComment Nothing "\"" --- -- "\""[^\n]*\n ++yy_mylinenumber; // BNFC: comment "\""; --- lexSingleComment :: Maybe String -> String -> Doc --- lexSingleComment ns c = --- "" <> cstring c <> "[^\\n]*\\n" --- <+> "++"<> text (fromMaybe "" ns)<>"yy_mylinenumber;" --- <+> "// BNFC: comment" <+> cstring c <> ";" - -- | Create a lexer rule for multi-lines comments. -- The first argument is -- an optional c++ namespace -- The second arguments is the pair of delimiter for the multi-lines comment: @@ -169,27 +150,27 @@ lexSingleComment _ c = -- with another. However this seems rare. -- -- >>> lexMultiComment Nothing ("{-", "-}") "COMMENT" --- "{-" BEGIN COMMENT; // BNFC: block comment "{-" "-}"; --- "-}" BEGIN YYINITIAL; +-- "{-" BEGIN COMMENT; // BNFC: block comment "{-" "-}"; +-- "-}" BEGIN INITIAL; -- . /* skip */; -- [\n] ++yy_mylinenumber; -- -- >>> lexMultiComment (Just "foo.") ("{-", "-}") "COMMENT" --- "{-" BEGIN COMMENT; // BNFC: block comment "{-" "-}"; --- "-}" BEGIN YYINITIAL; +-- "{-" BEGIN COMMENT; // BNFC: block comment "{-" "-}"; +-- "-}" BEGIN INITIAL; -- . /* skip */; -- [\n] ++foo.yy_mylinenumber; -- -- >>> lexMultiComment Nothing ("\"'", "'\"") "COMMENT" --- "\"'" BEGIN COMMENT; // BNFC: block comment "\"'" "'\""; --- "'\"" BEGIN YYINITIAL; +-- "\"'" BEGIN COMMENT; // BNFC: block comment "\"'" "'\""; +-- "'\"" BEGIN INITIAL; -- . /* skip */; -- [\n] ++yy_mylinenumber; lexMultiComment :: Maybe String -> (String, String) -> String -> Doc lexMultiComment ns (b,e) comment = vcat - [ "" <> cstring b <+> "BEGIN" <+> text comment <> ";" + [ "" <> cstring b <+> "BEGIN" <+> text comment <> ";" <+> "// BNFC: block comment" <+> cstring b <+> cstring e <> ";" - , commentTag <> cstring e <+> "BEGIN YYINITIAL;" + , commentTag <> cstring e <+> "BEGIN INITIAL;" , commentTag <> ". /* skip */;" , commentTag <> "[\\n] ++" <> text (fromMaybe "" ns) <> "yy_mylinenumber;" ] diff --git a/source/src/BNFC/Backend/Java/CFtoJLex15.hs b/source/src/BNFC/Backend/Java/CFtoJLex15.hs index 76a5b478..047d3cb7 100644 --- a/source/src/BNFC/Backend/Java/CFtoJLex15.hs +++ b/source/src/BNFC/Backend/Java/CFtoJLex15.hs @@ -125,25 +125,25 @@ cMacros cf = vcat $ concat -- | -- >>> lexSymbols JLexCup [("foo","bar")] --- foo { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } +-- foo { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } -- -- >>> lexSymbols JLexCup [("\\","bar")] --- \\ { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } +-- \\ { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } -- -- >>> lexSymbols JLexCup [("/","bar")] --- / { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } +-- / { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } -- -- >>> lexSymbols JFlexCup [("/","bar")] --- \/ { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } +-- \/ { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } -- -- >>> lexSymbols JFlexCup [("~","bar")] --- \~ { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } +-- \~ { return cf.newSymbol("", sym.bar, left_loc(), right_loc()); } -- lexSymbols :: JavaLexerParser -> SymEnv -> Doc lexSymbols jflex ss = vcat $ map transSym ss where transSym (s,r) = - "" <> text (escapeChars s) <> " { return cf.newSymbol(\"\", sym." + "" <> text (escapeChars s) <> " { return cf.newSymbol(\"\", sym." <> text r <> ", left_loc(), right_loc()); }" --Helper function that escapes characters in strings escapeChars :: String -> String @@ -157,12 +157,12 @@ restOfJLex jflex rp cf = vcat , ifC catString strStates , ifC catChar chStates , ifC catDouble - "{DIGIT}+\".\"{DIGIT}+(\"e\"(\\-)?{DIGIT}+)? { return cf.newSymbol(\"\", sym._DOUBLE_, left_loc(), right_loc(), new Double(yytext())); }" + "{DIGIT}+\".\"{DIGIT}+(\"e\"(\\-)?{DIGIT}+)? { return cf.newSymbol(\"\", sym._DOUBLE_, left_loc(), right_loc(), new Double(yytext())); }" , ifC catInteger - "{DIGIT}+ { return cf.newSymbol(\"\", sym._INTEGER_, left_loc(), right_loc(), new Integer(yytext())); }" + "{DIGIT}+ { return cf.newSymbol(\"\", sym._INTEGER_, left_loc(), right_loc(), new Integer(yytext())); }" , ifC catIdent - "{LETTER}{IDENT}* { return cf.newSymbol(\"\", sym._IDENT_, left_loc(), right_loc(), yytext().intern()); }" - , "[ \\t\\r\\n\\f] { /* ignore white space. */ }" + "{LETTER}{IDENT}* { return cf.newSymbol(\"\", sym._IDENT_, left_loc(), right_loc(), yytext().intern()); }" + , "[ \\t\\r\\n\\f] { /* ignore white space. */ }" , if jflex == JFlexCup then "<> { return cf.newSymbol(\"EOF\", sym.EOF, left_loc(), left_loc()); }" else "" @@ -175,14 +175,14 @@ restOfJLex jflex rp cf = vcat ifC :: TokenCat -> Doc -> Doc ifC cat s = if isUsedCat cf (TokenCat cat) then s else "" userDefTokens = vcat - [ "" <> text (printRegJLex jflex exp) + [ "" <> text (printRegJLex jflex exp) <+> "{ return cf.newSymbol(\"\", sym." <> text name <> ", left_loc(), right_loc(), yytext().intern()); }" | (name, exp) <- tokenPragmas cf ] strStates = vcat --These handle escaped characters in Strings. - [ "\"\\\"\" { left = left_loc(); yybegin(STRING); }" + [ "\"\\\"\" { left = left_loc(); yybegin(STRING); }" , "\\\\ { yybegin(ESCAPED); }" - , "\\\" { String foo = pstring; pstring = new String(); yybegin(YYINITIAL); return cf.newSymbol(\"\", sym._STRING_, left, right_loc(), foo.intern()); }" + , "\\\" { String foo = pstring; pstring = new String(); yybegin(INITIAL); return cf.newSymbol(\"\", sym._STRING_, left, right_loc(), foo.intern()); }" , ". { pstring += yytext(); }" , "\\r\\n|\\r|\\n { throw new Error(\"Unterminated string on line \" + left.getLine() " <> (if jflex == JFlexCup then "+ \" begining at column \" + left.getColumn()" else "") <> "); }" @@ -203,7 +203,7 @@ restOfJLex jflex rp cf = vcat else "" ] chStates = vcat --These handle escaped characters in Chars. - [ "\"'\" { left = left_loc(); yybegin(CHAR); }" + [ "\"'\" { left = left_loc(); yybegin(CHAR); }" , "\\\\ { yybegin(CHARESC); }" , "[^'] { yybegin(CHAREND); return cf.newSymbol(\"\", sym._CHAR_, left, right_loc(), new Character(yytext().charAt(0))); }" , "\\r\\n|\\r|\\n { throw new Error(\"Unterminated character literal on line \" + left.getLine() " <> @@ -221,7 +221,7 @@ restOfJLex jflex rp cf = vcat , if jflex == JFlexCup then "<> { throw new Error(\"Unterminated character literal at EOF, beginning at \" + left.getLine() + \":\" + left.getColumn()); }" else "" - , "\"'\" {yybegin(YYINITIAL);}" + , "\"'\" {yybegin(INITIAL);}" , "\\r\\n|\\r|\\n { throw new Error(\"Unterminated character literal on line \" + left.getLine() " <> (if jflex == JFlexCup then "+ \" beginning at column \" + left.getColumn()" else "") <> "); }" , if jflex == JFlexCup @@ -238,13 +238,13 @@ lexComments (m,s) = vcat $ concat -- | Create lexer rule for single-line comments. -- -- >>> lexSingleComment "--" --- "--"[^\n]* { /* skip */ } +-- "--"[^\n]* { /* skip */ } -- -- >>> lexSingleComment "\"" --- "\""[^\n]* { /* skip */ } +-- "\""[^\n]* { /* skip */ } lexSingleComment :: String -> Doc lexSingleComment c = - "" <> cstring c <> "[^\\n]* { /* skip */ }" + "" <> cstring c <> "[^\\n]* { /* skip */ }" -- | Create lexer rule for multi-lines comments. -- @@ -253,21 +253,21 @@ lexSingleComment c = -- with another. However this seems rare. -- -- >>> lexMultiComment ("{-", "-}") "COMMENT" --- "{-" { yybegin(COMMENT); } --- "-}" { yybegin(YYINITIAL); } +-- "{-" { yybegin(COMMENT); } +-- "-}" { yybegin(INITIAL); } -- . { /* skip */ } -- [\n] { /* skip */ } -- -- >>> lexMultiComment ("\"'", "'\"") "COMMENT" --- "\"'" { yybegin(COMMENT); } --- "'\"" { yybegin(YYINITIAL); } +-- "\"'" { yybegin(COMMENT); } +-- "'\"" { yybegin(INITIAL); } -- . { /* skip */ } -- [\n] { /* skip */ } -- lexMultiComment :: (String, String) -> String -> Doc lexMultiComment (b,e) comment = vcat - [ "" <> cstring b <+> "{ yybegin(" <> text comment <> "); }" - , commentTag <> cstring e <+> "{ yybegin(YYINITIAL); }" + [ "" <> cstring b <+> "{ yybegin(" <> text comment <> "); }" + , commentTag <> cstring e <+> "{ yybegin(INITIAL); }" , commentTag <> ". { /* skip */ }" , commentTag <> "[\\n] { /* skip */ }" ]