modified: EduNLP/SIF/tokenization/text/tokenization.py

modified: tests/test_tokenizer/test_tokenizer.py
bigdata-ustc · Mar 11, 2024 · ad7df8b · ad7df8b
1 parent a289a7a
commit ad7df8b
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 5 deletions.
diff --git a/EduNLP/SIF/tokenization/text/tokenization.py b/EduNLP/SIF/tokenization/text/tokenization.py
@@ -87,7 +87,7 @@ def tokenize(text,
     elif (tokenizer == 'spacy'):
         try:
             spacy_tokenizer = spacy.load(tok_model)
-        except LookupError:
+        except OSError:
             spacy.cli.download(tok_model)
             spacy_tokenizer = spacy.load(tok_model)
 

diff --git a/tests/test_tokenizer/test_tokenizer.py b/tests/test_tokenizer/test_tokenizer.py
@@ -78,10 +78,8 @@ def test_TokenizerBPE():
          ', ', '$', '25', '$ ', 'e', 'a', 'c', 'h', ', ', 'h', 'ow', ' m', 'an', 'y',
          ' ', 'are', ' ', 's', 'o', 'l', 'd']
     ]
-    tokenizer = get_tokenizer("pure_text",
-                             text_params={"tokenizer": 'bpe',
-                             "bpe_trainfile": "../../static/test_data/standard_luna_data.json",
-                             "stopwords": set(",?")})
+    tokenizer = get_tokenizer("pure_text", text_params={"tokenizer": 'bpe', "stopwords": set(",?"),
+                              "bpe_trainfile": "../../../../static/test_data/standard_luna_data.json"})
     tokens = tokenizer(items)
     ret = next(tokens)
     assert ret == ans