model improved

maker.py

@@ -6,6 +6,10 @@ url="https://github.com/UniversalDependencies/UD_Vietnamese-VTB"
 d=os.path.basename(url)
 os.system("test -d "+d+" || git clone --depth=1 "+url)
 os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
+url="https://github.com/datquocnguyen/VnDT"
+d=os.path.basename(url)
+os.system("test -d "+d+" || git clone --depth=1 "+url)
+os.system("for F in train dev test ; do cp "+d+"/*-gold-*-$F.conll pre-$F.conll ; done")
 class UDgoeswithDataset(object):
   def __init__(self,conllu,tokenizer):
     self.ids,self.tags,label=[],[],set()
@@ -17,7 +21,7 @@ class UDgoeswithDataset(object):
         if len(t)==10 and t[0].isdecimal():
           c.append(t)
         elif c!=[]:
-          v=tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
+          v=tokenizer([t[1].replace("_"," ") for t in c],add_special_tokens=False)["input_ids"]
           for i in range(len(v)-1,-1,-1):
             for j in range(1,len(v[i])):
               c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
@@ -43,13 +47,23 @@ class UDgoeswithDataset(object):
   __getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
 from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
 tkz=AutoTokenizer.from_pretrained(src)
+trainDS=UDgoeswithDataset("pre-train.conll",tkz)
+devDS=UDgoeswithDataset("pre-dev.conll",tkz)
+testDS=UDgoeswithDataset("pre-test.conll",tkz)
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model("tmpdir")
+tkz.save_pretrained("tmpdir")
 trainDS=UDgoeswithDataset("train.conllu",tkz)
 devDS=UDgoeswithDataset("dev.conllu",tkz)
 testDS=UDgoeswithDataset("test.conllu",tkz)
 lid=trainDS(devDS,testDS)
-cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
+cfg=AutoConfig.from_pretrained("tmpdir",num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
 arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
-trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained("tmpdir",config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
 trn.train()
 trn.save_model(tgt)
 tkz.save_pretrained(tgt)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53a888fb108192c80f8a0e937ca661808334b94961eee95832a70f7181b8e93c
+oid sha256:afbc383453f9f496264642c011fcd2cf8d8574ba2114d854a83d6565212e6494
 size 441306865