fix: vim lexer was marking \n as errors

Fixes #827
alecthomas · Aug 21, 2023 · 47ce9a2 · 47ce9a2
1 parent a20cd7e
commit 47ce9a2
Show file tree

Hide file tree

Showing 8 changed files with 585 additions and 109 deletions.
diff --git a/lexers/lexers_test.go b/lexers/lexers_test.go
@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	assert "github.com/alecthomas/assert/v2"
+	"github.com/alecthomas/assert/v2"
 	"github.com/alecthomas/repr"
 
 	"github.com/alecthomas/chroma/v2"
@@ -73,11 +73,16 @@ func FileTest(t *testing.T, lexer chroma.Lexer, actualFilename, expectedFilename
 		actual, err := chroma.Tokenise(lexer, nil, string(actualText))
 		assert.NoError(t, err)
 
-		if os.Getenv("RECORD") == "true" {
+		if os.Getenv("RECORD") != "" {
 			// Update the expected file with the generated output of this lexer
 			f, err := os.Create(expectedFilename)
 			defer f.Close() // nolint: gosec
 			assert.NoError(t, err)
+			for _, token := range actual {
+				if token.Type == chroma.Error {
+					t.Logf("Found Error token in lexer %s output: %s", lexer.Config().Name, repr.String(token))
+				}
+			}
 			assert.NoError(t, formatters.JSON.Format(f, nil, chroma.Literator(actual...)))
 		} else {
 			// Read expected JSON into token slice.
@@ -87,10 +92,14 @@ func FileTest(t *testing.T, lexer chroma.Lexer, actualFilename, expectedFilename
 			err = json.NewDecoder(r).Decode(&expected)
 			assert.NoError(t, err)
 
-			// Equal?
-			assert.Equal(t,
-				repr.String(expected, repr.Indent(" ")),
-				repr.String(actual, repr.Indent(" ")))
+			assert.Equal(t, expected, actual)
+
+			// Check for error tokens.
+			for _, token := range actual {
+				if token.Type == chroma.Error {
+					t.Logf("Found Error token in lexer %s output: %s", lexer.Config().Name, repr.String(token))
+				}
+			}
 		}
 	})
 }

diff --git a/lexers/testdata/hlsl.expected b/lexers/testdata/hlsl.expected
@@ -1,40 +1,40 @@
-[
-  {"type":"KeywordType","value":"float4"},
-  {"type":"Text","value":" "},
-  {"type":"Name","value":"vertex"},
-  {"type":"Punctuation","value":"("},
-  {"type":"KeywordType","value":"float3"},
-  {"type":"Text","value":" "},
-  {"type":"Name","value":"position"},
-  {"type":"Punctuation","value":")"},
-  {"type":"Text","value":" "},
-  {"type":"Punctuation","value":"{"},
-  {"type":"Text","value":"\n    "},
-  {"type":"Keyword","value":"return"},
-  {"type":"Text","value":" "},
-  {"type":"NameBuiltin","value":"mul"},
-  {"type":"Punctuation","value":"("},
-  {"type":"Name","value":"projection"},
-  {"type":"Punctuation","value":","},
-  {"type":"Text","value":" "},
-  {"type":"NameBuiltin","value":"mul"},
-  {"type":"Punctuation","value":"("},
-  {"type":"Name","value":"view"},
-  {"type":"Punctuation","value":","},
-  {"type":"Text","value":" "},
-  {"type":"NameBuiltin","value":"mul"},
-  {"type":"Punctuation","value":"("},
-  {"type":"Name","value":"model"},
-  {"type":"Punctuation","value":","},
-  {"type":"Text","value":" "},
-  {"type":"KeywordType","value":"float4"},
-  {"type":"Punctuation","value":"("},
-  {"type":"Name","value":"position"},
-  {"type":"Punctuation","value":","},
-  {"type":"Text","value":" "},
-  {"type":"LiteralNumberFloat","value":"1.0"},
-  {"type":"Punctuation","value":"))));"},
-  {"type":"Text","value":"\n"},
-  {"type":"Punctuation","value":"}"},
-  {"type":"Text","value":"\n"}
-]
+[
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"vertex"},
+  {"type":"Punctuation","value":"("},
+  {"type":"KeywordType","value":"float3"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"mul"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"projection"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"mul"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"view"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"mul"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"model"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1.0"},
+  {"type":"Punctuation","value":"))));"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"}
+]
diff --git a/lexers/testdata/natural.expected b/lexers/testdata/natural.expected
@@ -16,9 +16,9 @@
   {"type":"Other","value":"(A10)"},
   {"type":"Text","value":" "},
   {"type":"Keyword","value":"INIT"},
-  {"type":"Operator","value":"<"},
+  {"type":"Operator","value":"\u003c"},
   {"type":"LiteralStringSingle","value":"'Hello'"},
-  {"type":"Operator","value":">"},
+  {"type":"Operator","value":"\u003e"},
   {"type":"Text","value":"\n"},
   {"type":"LiteralNumberInteger","value":"1"},
   {"type":"Text","value":" "},
@@ -48,14 +48,12 @@
   {"type":"Text","value":" "},
   {"type":"NameVariable","value":"#VAR-NAME-1"},
   {"type":"Text","value":"\n\n"},
-
   {"type":"Keyword","value":"DEFINE"},
   {"type":"Text","value":" "},
   {"type":"Keyword","value":"SUBROUTINE"},
   {"type":"Text","value":" "},
   {"type":"NameFunction","value":"MY-SUBROUTINE"},
   {"type":"Text","value":"\n"},
-
   {"type":"Keyword","value":"FOR"},
   {"type":"Text","value":" "},
   {"type":"NameVariable","value":"#I"},
@@ -68,17 +66,13 @@
   {"type":"Text","value":" "},
   {"type":"LiteralNumberInteger","value":"10"},
   {"type":"Text","value":"\n"},
-
   {"type":"Keyword","value":"ESCAPE"},
   {"type":"Text","value":" "},
   {"type":"Keyword","value":"MODULE"},
   {"type":"Text","value":"\n"},
-
   {"type":"Keyword","value":"END-FOR"},
   {"type":"Text","value":"\n"},
-
   {"type":"Keyword","value":"END-SUBROUTINE"},
-
   {"type":"Text","value":"\n\n"},
   {"type":"Keyword","value":"END"},
   {"type":"Text","value":"\n\n"}

diff --git a/lexers/testdata/vhs.expected b/lexers/testdata/vhs.expected
@@ -1,54 +1,54 @@
 [
-	{"type":"Comment","value":"# Where should we write the GIF?"},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Output"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"LiteralString","value":"demo.gif"},
-	{"type":"TextWhitespace","value":"\n\n"},
-	{"type":"Comment","value":"# Set up a 1200x600 terminal with 46px font."},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Set"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"NameBuiltin","value":"FontSize"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"LiteralNumber","value":"46"},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Set"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"NameBuiltin","value":"Width"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"LiteralNumber","value":"1200"},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Set"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"NameBuiltin","value":"Height"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"LiteralNumber","value":"600"},
-	{"type":"TextWhitespace","value":"\n\n"},
-	{"type":"Comment","value":"# Type a command in the terminal."},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Type"},
-	{"type":"Punctuation", "value": "@"},
-	{"type":"LiteralNumber", "value": "100"},
-	{"type":"Text", "value": "ms"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"LiteralStringDouble","value":"\"echo 'Welcome to VHS!'\""},
-	{"type":"TextWhitespace","value":"\n\n"},
-	{"type":"Comment","value":"# Pause for dramatic effect..."},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Sleep"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"LiteralNumber","value":"500"},
-	{"type":"Text","value":"ms"},
-	{"type":"TextWhitespace","value":"\n\n"},
-	{"type":"Comment","value":"# Run the command by pressing enter."},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Enter"},
-	{"type":"TextWhitespace","value":"\n\n"},
-	{"type":"Comment","value":"# Admire the output for a bit."},
-	{"type":"TextWhitespace","value":"\n"},
-	{"type":"Keyword","value":"Sleep"},
-	{"type":"TextWhitespace","value":" "},
-	{"type":"LiteralNumber","value":"5"},
-	{"type":"Text","value":"s"}
+  {"type":"Comment","value":"# Where should we write the GIF?"},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Output"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralString","value":"demo.gif"},
+  {"type":"TextWhitespace","value":"\n\n"},
+  {"type":"Comment","value":"# Set up a 1200x600 terminal with 46px font."},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Set"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameBuiltin","value":"FontSize"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumber","value":"46"},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Set"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameBuiltin","value":"Width"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumber","value":"1200"},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Set"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameBuiltin","value":"Height"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumber","value":"600"},
+  {"type":"TextWhitespace","value":"\n\n"},
+  {"type":"Comment","value":"# Type a command in the terminal."},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Type"},
+  {"type":"Punctuation","value":"@"},
+  {"type":"LiteralNumber","value":"100"},
+  {"type":"Text","value":"ms"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"echo 'Welcome to VHS!'\""},
+  {"type":"TextWhitespace","value":"\n\n"},
+  {"type":"Comment","value":"# Pause for dramatic effect..."},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Sleep"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumber","value":"500"},
+  {"type":"Text","value":"ms"},
+  {"type":"TextWhitespace","value":"\n\n"},
+  {"type":"Comment","value":"# Run the command by pressing enter."},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Enter"},
+  {"type":"TextWhitespace","value":"\n\n"},
+  {"type":"Comment","value":"# Admire the output for a bit."},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Keyword","value":"Sleep"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumber","value":"5"},
+  {"type":"Text","value":"s"}
 ]
diff --git a/lexers/testdata/vim.actual b/lexers/testdata/vim.actual
@@ -0,0 +1,88 @@
+execute pathogen#infect()
+
+syntax on
+colorscheme swapoff
+
+" Turn off status line.
+set laststatus=0
+
+" Turn off annoying beeps
+set vb
+set noeb
+set vb t_vb=
+
+set mouse=r
+set visualbell
+
+" Force latin1 terminal encoding?
+set fileencoding=utf-8
+set encoding=utf-8
+set termencoding=utf-8
+
+" Allows ~<motion>
+set tildeop
+
+" Stupid backspace
+set backspace=eol,indent,start
+
+" 99% of the time, wrap is shit
+set nowrap
+
+" Map U to 'redo', which I like better than U being 'undo changes to this line'
+map U <C-R>
+
+" Indentation defaults
+set ts=2 sts=2 sw=2 et
+
+" Abbreviate status messages
+set shortmess=aTI
+
+" No 'Press ENTER...' message when using man
+map K K<CR>
+
+" Filetypes that need real tabs
+autocmd FileType make :set ts=4 noet nolist
+
+" Set some formatoptions
+set formatoptions=tcrqn
+
+" Incremental search is good
+set incsearch
+set hlsearch
+hi MatchParen cterm=NONE ctermfg=LightGreen ctermbg=NONE
+
+" Disable highlighted search on redraw
+map <silent> <C-l> :nohlsearch<CR>:redraw!<CR>
+
+autocmd Syntax * syntax sync fromstart
+
+filetype plugin on
+filetype indent on
+
+" Always show cursor position
+set ruler
+
+" Fucking visual-select copying to clipboard! WTF!
+set clipboard=
+
+" C-d when file-completing
+set wildmenu
+
+highlight SpellBad cterm=NONE ctermfg=darkred ctermbg=none
+
+" Show trailing whitepace and spaces before a tab:
+highlight ExtraWhitespace cterm=NONE ctermbg=red guibg=red
+autocmd Syntax * syn match ExtraWhitespace /\s\+$\| \+\ze\t/
+
+" Round shifting to nearest shiftwidth
+set shiftround
+
+" Insert blanks according to shiftwidth
+set smarttab
+
+" Disable toolbar
+set go-=T
+
+set wildignore=*.o,*.a,*.so,*.pyc,*~,*.class,build/*,build-*,tags,cscope*,third_party/*,java/*,users/*,data/*,3rdparty/*,*.jar,target/*,dist/*
+
+autocmd BufRead,BufNewFile *.bk set filetype=python