model improved

config.json

@@ -371,7 +371,7 @@
   "summary_use_proj": true,
   "tokenizer_class": "PreTrainedTokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.42.4",
+  "transformers_version": "4.44.2",
   "use_cache": true,
   "vocab_size": 32000
 }

maker.sh → maker.py

@@ -1,22 +1,17 @@
-#! /bin/sh
-S=abeja/gpt2-large-japanese
-T=KoichiYasuoka/abeja-gpt2-large-japanese-ud-causal
-U=https://github.com/UniversalDependencies/UD_Japanese-GSDLUW
-D=`basename $U`
-test -d $D || git clone --depth=1 $U
-for F in train dev test
-do cp $D/*-$F.conllu $F.conllu
-done
+#! /usr/bin/python3
+src="abeja/gpt2-large-japanese"
+tgt="KoichiYasuoka/abeja-gpt2-large-japanese-ud-causal"
+url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW"
 
-TMPA=./maker$$a.py
-( echo '#! /usr/bin/python3'
-  echo 'src="'$S'"'
-  cat << 'EOF'
-import json
-from transformers import AutoTokenizer
+import os,json,unicodedata
+from transformers import AutoTokenizer,PreTrainedTokenizerFast,AutoConfig,GPT2ForTokenClassification,DefaultDataCollator,TrainingArguments,Trainer
+d=os.path.basename(url)
+os.system("test -d "+d+" || git clone --depth=1 "+url)
+os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
 tkz=AutoTokenizer.from_pretrained(src,add_prefix_space=False,legacy=False,model_max_length=1280)
 tkz.save_pretrained("tmpdir")
 d=json.loads(tkz.backend_tokenizer.to_str())
+tkz.backend_tokenizer.from_str(json.dumps(d)).save("tmpdir/oldtokenizer.json")
 form=set()
 with open("train.conllu","r",encoding="utf-8") as r:
   for s in r:
@@ -27,22 +22,14 @@ for t in d["model"]["vocab"]:
   if t[0] not in form:
     t[1]*=len(t[0])
 tkz.backend_tokenizer.from_str(json.dumps(d)).save("tmpdir/tokenizer.json")
-EOF
-) > $TMPA
-chmod 755 $TMPA
-$TMPA
-
-TMPB=./maker$$b.py
-( echo '#! /usr/bin/env deepspeed'
-  echo 'src="'$S'"'
-  echo 'tgt="'$T'"'
-  cat << 'EOF'
-from transformers import PreTrainedTokenizerFast,AutoConfig,GPT2ForTokenClassification,DefaultDataCollator,TrainingArguments,Trainer
+ntk=PreTrainedTokenizerFast.from_pretrained("tmpdir")
+otk=PreTrainedTokenizerFast.from_pretrained("tmpdir",tokenizer_file="tmpdir/oldtokenizer.json")
 
 class UDCausalDataset(object):
-  def __init__(self,conllu,tokenizer,embeddings=None):
+  def __init__(self,conllu,tokenizer,oldtokenizer=None,embeddings=None):
     self.conllu=open(conllu,"r",encoding="utf-8")
     self.tokenizer=tokenizer
+    self.oldtokenizer=oldtokenizer if oldtokenizer else tokenizer
     self.embeddings=embeddings
     self.max_tokens=3
     self.seeks=[(0,0)]
@@ -87,8 +74,8 @@ class UDCausalDataset(object):
         if w[0].isdecimal():
           upos.append(w[3] if w[5]=="_" else w[3]+"|"+w[5])
           deps.append((int(w[6]),w[7]))
-    v=self.tokenizer(form,add_special_tokens=False)
     if t==0:
+      v=self.tokenizer(form,add_special_tokens=False)
       i,u=[],[]
       for j,(x,y) in enumerate(zip(v["input_ids"],upos)):
         if x!=[]:
@@ -98,6 +85,7 @@ class UDCausalDataset(object):
       pad=self.tokenizer.pad_token_id
     else:
       import torch
+      v=self.oldtokenizer(form,add_special_tokens=False)
       m=[]
       for x in v["input_ids"]:
         if x==[]:
@@ -125,23 +113,16 @@ class UDCausalDataset(object):
       upos=u[0:self.max_tokens]
     return {"inputs_embeds":emb[ids,:],"labels":[self.label2id[p] for p in upos]}
 
-tkz=PreTrainedTokenizerFast.from_pretrained("tmpdir")
-trainDS=UDCausalDataset("train.conllu",tkz)
-devDS=UDCausalDataset("dev.conllu",tkz)
-testDS=UDCausalDataset("test.conllu",tkz)
+trainDS=UDCausalDataset("train.conllu",ntk,otk)
+devDS=UDCausalDataset("dev.conllu",ntk,otk)
+testDS=UDCausalDataset("test.conllu",ntk,otk)
 lid=trainDS(devDS,testDS)
 cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
 mdl=GPT2ForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True)
-trainDS.embeddings=mdl.get_input_embeddings().weight.detach().cpu()
+trainDS.embeddings=mdl.get_input_embeddings().weight
 trainDS.max_tokens=min(trainDS.max_tokens,cfg.max_position_embeddings)
-dsp={"fp16":{"enabled":"auto"},"optimizer":{"type":"AdamW"},"scheduler":{"type":"WarmupLR","params":{}},"train_batch_size":"auto","train_micro_batch_size_per_gpu":"auto","zero_optimization":{"stage":3,"offload_optimizer":{"device":"cpu","pin_memory":True},"offload_param":{"device":"cpu","pin_memory":True},"overlap_comm":True,"contiguous_gradients":True,"reduce_bucket_size":"auto","stage3_prefetch_bucket_size":"auto","stage3_param_persistence_threshold":"auto","stage3_gather_16bit_weights_on_model_save":True}}
-arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=16,deepspeed=dsp,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=16,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
 trn=Trainer(args=arg,data_collator=DefaultDataCollator(),model=mdl,train_dataset=trainDS)
 trn.train()
 trn.save_model(tgt)
-tkz.save_pretrained(tgt)
-EOF
-) > $TMPB
-chmod 755 $TMPB
-$TMPB
-exit
+ntk.save_pretrained(tgt)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5771f96d35bf2299e87a19725f670365d927bfb052813568e50f4c60906304d
-size 3003627106
+oid sha256:43a978eb2923908e13abf4f6698881dd6fe29375c2a99d06877883dd31a28014
+size 3003633250

ud.py

@@ -1,5 +1,10 @@
 import numpy
-from transformers import TokenClassificationPipeline
+from transformers import TokenClassificationPipeline,AutoTokenizer
+try:
+  from transformers.utils import cached_file
+except:
+  from transformers.file_utils import cached_path,hf_bucket_url
+  cached_file=lambda x,y:os.path.join(x,y) if os.path.isdir(x) else cached_path(hf_bucket_url(x,y))
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -42,6 +47,7 @@ class UniversalDependenciesCausalPipeline(BellmanFordTokenClassificationPipeline
   def __init__(self,**kwargs):
     kwargs["aggregation_strategy"]="simple"
     super().__init__(**kwargs)
+    self.oldtokenizer=AutoTokenizer.from_pretrained(self.tokenizer.name_or_path,tokenizer_file=cached_file(self.tokenizer.name_or_path,"oldtokenizer.json"))
     x=self.model.config.label2id
     self.root=numpy.full((len(x)),numpy.nan)
     self.left_arc=numpy.full((len(x)),numpy.nan)
@@ -87,7 +93,7 @@ class UniversalDependenciesCausalPipeline(BellmanFordTokenClassificationPipeline
       if d[i].strip()=="":
         d.pop(i)
         w.pop(i)
-    v=self.tokenizer(d,add_special_tokens=False)
+    v=self.oldtokenizer(d,add_special_tokens=False)
     e=self.model.get_input_embeddings().weight
     m=[]
     for x in v["input_ids"]: