Merge pull request #5 from todd-cook/fix_tokenizer_imports

Fix broken tokenizer imports

case_studies/contextual_nearest_neighbors/scripts/gen_berts.py

@@ -1,6 +1,6 @@
 import argparse, sys
-from cltk.tokenize.word import WordTokenizer
-from cltk.tokenize.latin.sentence import SentenceTokenizer
+from cltk.tokenizers.lat.lat import LatinWordTokenizer as WordTokenizer
+from cltk.tokenizers.lat.lat import LatinPunktSentenceTokenizer as SentenceTokenizer
 from tensor2tensor.data_generators import text_encoder
 import numpy as np
 import torch
@@ -249,7 +249,7 @@ def tokenize(self, text):
 def convert_to_toks(sents):
 
 	sent_tokenizer = SentenceTokenizer()
-	word_tokenizer = WordTokenizer('latin')
+	word_tokenizer = WordTokenizer()
 
 	all_sents=[]
 

case_studies/contextual_nearest_neighbors/scripts/gen_berts_to_file.py

@@ -1,6 +1,6 @@
 import argparse, sys
-from cltk.tokenize.word import WordTokenizer
-from cltk.tokenize.latin.sentence import SentenceTokenizer
+from cltk.tokenizers.lat.lat import LatinWordTokenizer as WordTokenizer
+from cltk.tokenizers.lat.lat import LatinPunktSentenceTokenizer as SentenceTokenizer
 from tensor2tensor.data_generators import text_encoder
 import numpy as np
 import torch
@@ -64,7 +64,7 @@ def tokenize(self, text):
 
 def read_file(filename):
 	sent_tokenizer = SentenceTokenizer()
-	word_tokenizer = WordTokenizer('latin')
+	word_tokenizer = WordTokenizer()
 
 	all_sents=[]
 	with open(filename, encoding="utf-8") as file:

case_studies/infilling/scripts/check_dupes_emendations.py

@@ -1,8 +1,7 @@
 import sys, re
-from cltk.tokenize.word import WordTokenizer
-from cltk.tokenize.latin.sentence import SentenceTokenizer
+from cltk.tokenizers.lat.lat import LatinWordTokenizer as WordTokenizer
 
-word_tokenizer = WordTokenizer('latin')
+word_tokenizer = WordTokenizer()
 
 def filt(text):
 	text=re.sub("<", "", text)
@@ -59,4 +58,4 @@ def search(filename, tests):
 				print("dupe\t%s\t%s\t%s\t%s" % (origq, left, target, right))
 
 tests=read_test(sys.argv[1])
-search(sys.argv[2], tests)
+search(sys.argv[2], tests)

case_studies/infilling/scripts/predict_word.py

@@ -5,11 +5,9 @@
 from tensor2tensor.data_generators import text_encoder
 import torch
 import numpy as np
-from cltk.tokenize.word import WordTokenizer
-from cltk.tokenize.latin.sentence import SentenceTokenizer
+from cltk.tokenizers.lat.lat import LatinWordTokenizer as WordTokenizer
 from torch import nn
 import random
-from random import randrange
 
 random.seed(1)
 
@@ -20,7 +18,7 @@ def __init__(self, encoder):
 		self.vocab={}
 		self.reverseVocab={}
 		self.encoder=encoder
-		self.word_tokenizer = WordTokenizer('latin')
+		self.word_tokenizer = WordTokenizer()
 
 		self.vocab["[PAD]"]=0
 		self.vocab["[UNK]"]=1
@@ -241,5 +239,3 @@ def proc_file(filename, wp_tokenizer, model):
 	model.to(device)
 
 	proc_file(dataFile, wp_tokenizer, model)
-
-

case_studies/wsd/scripts/create_wsd_data.py

@@ -1,8 +1,8 @@
 import sys, re
-from cltk.tokenize.word import WordTokenizer
+from cltk.tokenizers.lat.lat import LatinWordTokenizer as WordTokenizer
 from unidecode import unidecode
 
-word_tokenizer = WordTokenizer('latin')
+word_tokenizer = WordTokenizer()
 
 def read_lemmas(filename):
 	lemmadict={}

requirements.txt

@@ -1,6 +1,6 @@
 pygame==2.0.0.dev6
 beautifulsoup4==4.9.1
-cltk==0.1.121
+cltk==1.0.15
 future==0.18.2
 numpy==1.18.5
 tensor2tensor==1.15.7

scripts/gen_berts.py

@@ -1,6 +1,6 @@
 import argparse, sys
-from cltk.tokenize.word import WordTokenizer
-from cltk.tokenize.latin.sentence import SentenceTokenizer
+from cltk.tokenizers.lat.lat import LatinWordTokenizer as WordTokenizer
+from cltk.tokenizers.lat.lat import LatinPunktSentenceTokenizer as SentenceTokenizer
 from tensor2tensor.data_generators import text_encoder
 import numpy as np
 import torch
@@ -235,7 +235,7 @@ def tokenize(self, text):
 def convert_to_toks(sents):
 
 	sent_tokenizer = SentenceTokenizer()
-	word_tokenizer = WordTokenizer('latin')
+	word_tokenizer = WordTokenizer()
 
 	all_sents=[]