hplt_gpt_bert_base_3_0_eng_Latn-UDは「don't」をどうトークナイズするのか

HPLT 3.0をチェックしていたところ、UD-Parsersという係り受け解析モジュール群がリリースされていたことに気づいた。とりあえず、英語版UD-Parserを動かしてみよう。Google Colaboratoryだと、こんな感じ。

!pip install 'transformers<5' ufal.chu_liu_edmonds deplacy
txt=["I saw a horse yesterday which had no name but today I don't"]
mdl="HPLT/hplt_gpt_bert_base_3_0_eng_Latn-UD"
import os,torch
d=os.path.basename(mdl)
!test -d {d} || git clone --depth=1 https://huggingface.co/{mdl}
%cd {d}
from lemma_rule import apply_lemma_rule
from preprocessor import Preprocessor
from utils import load_model
tokenizer,model,predict_model=load_model(mdl.replace("-UD",""))
preprocessor=Preprocessor(tokenizer=tokenizer)
batch=preprocessor.preprocess(txt)
with torch.no_grad():
  lemma_p,upos_p,xpos_p,feats_p,dummy1,dummy2,dep_p,head_p=predict_model(batch["subwords"],batch["alignment"],batch["subword_lengths"],batch["word_lengths"])
dts=model["dataset"]
doc=""
for i in range(len(txt)):
  doc+="# text = "+txt[i].replace("\n"," ")+"\n"
  for j,form in enumerate(batch["words"][i]):
    doc+="\t".join([str(j+1),form,apply_lemma_rule(form,{t:dts.lemma_vocab[t].get(p[i,j,:].argmax().item(),dts.lemma_vocab[t][-1]) for t,p in lemma_p.items()}),dts.upos_vocab[upos_p[i,j,:].argmax().item()],dts.xpos_vocab[xpos_p[i,j,:].argmax().item()],dts.feats_vocab[feats_p[i,j,:].argmax().item()],str(head_p[i,j].item()),dts.arc_dep_vocab[dep_p[i,j,:].argmax().item()],"_","_"])+"\n"
  doc+="\n"
%cd ..
import deplacy
deplacy.serve(doc,port=None)

「I saw a horse yesterday which had no name but today I don't」を係り受け解析してみたところ、私(安岡孝一)の手元では以下の結果が出力された。

# text = I saw a horse yesterday which had no name but today I don't
1	I	I	PRON	PRP	Case=Nom|Number=Sing|Person=1|PronType=Prs	2	nsubj	_	_
2	saw	see	VERB	VBD	Mood=Ind|Number=Sing|Person=1|Tense=Past|VerbForm=Fin	0	root	_	_
3	a	a	DET	DT	Definite=Ind|PronType=Art	4	det	_	_
4	horse	horse	NOUN	NN	Number=Sing	2	obj	_	_
5	yesterday	yesterday	NOUN	NN	Number=Sing	2	obl:tmod	_	_
6	which	which	PRON	WDT	PronType=Rel	7	nsubj	_	_
7	had	have	VERB	VBD	Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin	4	acl:relcl	_	_
8	no	no	DET	DT	PronType=Neg	9	det	_	_
9	name	name	NOUN	NN	Number=Sing	7	obj	_	_
10	but	but	CCONJ	CC	_	13	cc	_	_
11	today	today	NOUN	NN	Number=Sing	13	obl:tmod	_	_
12	I	I	PRON	PRP	Case=Nom|Number=Sing|Person=1|PronType=Prs	13	nsubj	_	_
13	don	don	AUX	VBP	Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin	7	conj	_	_
14	'	o'	PART	RB	_	13	advmod	_	_
15	t	ot	PART	RB	_	0	root	_	_

うーん、残念。「don't」は「do」「n't」とトークナイズしなきゃいけないのに、誤って「don」「'」「t」とトークナイズしてしまったために、その部分のリンクが壊れてしまっている。日本語版UD-Parserも試してみよう。Google Colaboratoryだと、こんな感じ。

!pip install 'transformers<5' ufal.chu_liu_edmonds deplacy
txt=["国境の長いトンネルを抜けると雪国であった"]
mdl="HPLT/hplt_gpt_bert_base_3_0_jpn_Jpan-UD"
import os,torch
d=os.path.basename(mdl)
!test -d {d} || git clone --depth=1 https://huggingface.co/{mdl}
%cd {d}
from lemma_rule import apply_lemma_rule
from preprocessor import Preprocessor
from utils import load_model
tokenizer,model,predict_model=load_model(mdl.replace("-UD",""))
preprocessor=Preprocessor(tokenizer=tokenizer)
batch=preprocessor.preprocess(txt)
with torch.no_grad():
  lemma_p,upos_p,xpos_p,feats_p,dummy1,dummy2,dep_p,head_p=predict_model(batch["subwords"],batch["alignment"],batch["subword_lengths"],batch["word_lengths"])
dts=model["dataset"]
doc=""
for i in range(len(txt)):
  doc+="# text = "+txt[i].replace("\n"," ")+"\n"
  for j,form in enumerate(batch["words"][i]):
    doc+="\t".join([str(j+1),form,apply_lemma_rule(form,{t:dts.lemma_vocab[t].get(p[i,j,:].argmax().item(),dts.lemma_vocab[t][-1]) for t,p in lemma_p.items()}),dts.upos_vocab[upos_p[i,j,:].argmax().item()],dts.xpos_vocab[xpos_p[i,j,:].argmax().item()],dts.feats_vocab[feats_p[i,j,:].argmax().item()],str(head_p[i,j].item()),dts.arc_dep_vocab[dep_p[i,j,:].argmax().item()],"_","_"])+"\n"
  doc+="\n"
%cd ..
import deplacy
deplacy.serve(doc,port=None)

「国境の長いトンネルを抜けると雪国であった」を係り受け解析してみたところ、私の手元では、以下のエラーでコケてしまった。

/content/hplt_gpt_bert_base_3_0_jpn_Jpan-UD/model.py in forward(self, head_x, dep_x, lengths, head_gold)
    128             head_gold = []
    129             for i, length in enumerate(lengths.tolist()):
--> 130                 head = self.max_spanning_tree(head_logp[i, :length, :length])
    131                 head = head + ((head_x.size(1) - 1) - len(head)) * [0]
    132                 head_gold.append(torch.tensor(head))

/content/hplt_gpt_bert_base_3_0_jpn_Jpan-UD/model.py in max_spanning_tree(self, weight_matrix)
    178                 print(" ".join([str(x) for x in matrix[i]]))
    179 
--> 180         assert best_parents is not None, f"{best_parents}\n{print_whole_matrix(weight_matrix)}"
    181         return best_parents
    182 

AssertionError: None
None

単語間に空白がない言語については、Preprocessorがマトモに動作しないようだ。とりあえずissue上げておいたけど、そもそもトークナイザの問題もあるので、さすがに直すの難しいだろうなあ。