exclude pytextspan

README.md

@@ -28,4 +28,4 @@ nlp=pipeline("universal-dependencies","KoichiYasuoka/deberta-base-japanese-juman
 print(nlp("全学年にわたって小学校の国語の教科書に挿し絵が用いられている"))
 ```
 
-[fugashi](https://pypi.org/project/fugashi) and [pytextspan](https://pypi.org/project/pytextspan) are required.
+[fugashi](https://pypi.org/project/fugashi) is required.

ud.py

@@ -68,10 +68,17 @@ class UniversalDependenciesPipeline(TokenClassificationPipeline):
 
 class MecabPreTokenizer(MecabTokenizer):
   def mecab_split(self,i,normalized_string):
-    import textspan
     t=str(normalized_string)
-    k=self.tokenize(t)
-    return [normalized_string[s:e] for c in textspan.get_original_spans(k,t) for s,e in c]
+    z=[]
+    e=0
+    for c in self.tokenize(t):
+      s=t.find(c,e)
+      if s<0:
+        z.append((0,0))
+      else:
+        e=s+len(c)
+        z.append((s,e))
+    return [normalized_string[s:e] for s,e in z]
   def pre_tokenize(self,pretok):
     pretok.split(self.mecab_split)