* Fix infixed commas in tokenizer, re Issue #326. Need to benchmark o…

…n empirical data, to make sure this doesn't break other cases.
explosion · Apr 14, 2016 · 6f82065 · 6f82065
1 parent 0f957dd
commit 6f82065
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/lang_data/en/infix.txt b/lang_data/en/infix.txt
@@ -3,3 +3,4 @@
 (?<=[a-zA-Z])-(?=[a-zA-z])
 (?<=[a-zA-Z])--(?=[a-zA-z])
 (?<=[0-9])-(?=[0-9])
+(?<=[A-Za-z]),(?=[A-Za-z])
diff --git a/spacy/tests/tokenizer/test_infix.py b/spacy/tests/tokenizer/test_infix.py
@@ -47,3 +47,10 @@ def test_double_hyphen(en_tokenizer):
     assert tokens[8].text == u'--'
     assert tokens[9].text == u'people'
 
+
+def test_infix_comma(en_tokenizer):
+    # Re issue #326
+    tokens = en_tokenizer(u'Hello,world')
+    assert tokens[0].text == u'Hello'
+    assert tokens[1].text == u','
+    assert tokens[2].text == u'world'