initial release

maker.py

@@ -0,0 +1,53 @@
+#! /usr/bin/python3
+src="KoichiYasuoka/deberta-large-chinese-erlangshen-upos"
+tgt="KoichiYasuoka/deberta-large-chinese-erlangshen-ud-goeswith"
+import os
+for d in ["UD_Chinese-GSD","UD_Chinese-GSDSimp"]:
+  os.system("test -d "+d+" || git clone --depth=1 https://github.com/UniversalDependencies/"+d)
+os.system("for F in train dev test ; do cat UD_Chinese-*/*-$F.conllu > $F.conllu ; done")
+class UDgoeswithDataset(object):
+  def __init__(self,conllu,tokenizer):
+    self.ids,self.tags,label=[],[],set()
+    with open(conllu,"r",encoding="utf-8") as r:
+      cls,sep,msk=tokenizer.cls_token_id,tokenizer.sep_token_id,tokenizer.mask_token_id
+      dep,c="-|_|dep",[]
+      for s in r:
+        t=s.split("\t")
+        if len(t)==10 and t[0].isdecimal():
+          c.append(t)
+        elif c!=[]:
+          v=tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
+          for i in range(len(v)-1,-1,-1):
+            for j in range(1,len(v[i])):
+              c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
+          y=["0"]+[t[0] for t in c]
+          h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
+          p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[])
+          self.ids.append([cls]+v+[sep])
+          self.tags.append([dep]+p+[dep])
+          label=set(sum([self.tags[-1],list(label)],[]))
+          for i,k in enumerate(v):
+            self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
+            self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
+          c=[]
+    self.label2id={l:i for i,l in enumerate(sorted(label))}
+  def __call__(*args):
+    label=set(sum([list(t.label2id) for t in args],[]))
+    lid={l:i for i,l in enumerate(sorted(label))}
+    for t in args:
+      t.label2id=lid
+    return lid
+  __len__=lambda self:len(self.ids)
+  __getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
+from transformers import BertTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
+tkz=BertTokenizer.from_pretrained(src,model_max_length=512)
+trainDS=UDgoeswithDataset("train.conllu",tkz)
+devDS=UDgoeswithDataset("dev.conllu",tkz)
+testDS=UDgoeswithDataset("test.conllu",tkz)
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=8,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model(tgt)
+tkz.save_pretrained(tgt)