日本語ModernBERTをJMTEBで測る

2月13日の記事の続きだが、modernbert-ja-130mをJMTEBで評価してみた。python3.9のlinux環境だと、こんな感じ。

#! /bin/sh
model="sbintuitions/modernbert-ja-130m"
pip3.9 install sentence-transformers accelerate datasets jsonnet jsonargparse smart-open openai loguru tiktoken triton
test -d JMTEB || git clone --depth=1 https://github.com/sbintuitions/JMTEB
NAWK=nawk
if [ -x /usr/bin/gawk ]
then NAWK=/usr/bin/gawk
elif [ -x /usr/bin/mawk ]
then NAWK=/usr/bin/mawk
fi
find JMTEB/src -type f -print |
( while read F
  do T=`echo $F | sed 's?/src/?/src9/?'`
     mkdir -p `dirname $T`
     case $T in
     *.py) $NAWK '
{
  if(f!=1){
    if($0!~/__future__/){
      printf("from typing import Union\n");
      f=1;
    }
  }
  s=$0;
  while(match(s,/[A-Za-z]+(\[[A-Za-z ",]+\])?( \| [A-Za-z]+(\[[A-Za-z ",]+\])?)+/)>0){
    u=substr(s,RSTART,RLENGTH)
    gsub(/\|/,",",u);
    s=substr(s,1,RSTART-1)"Union["u"]"substr(s,RSTART+RLENGTH);
  }
  print s;
}' $F > $T ;;
     *) cp $F $T ;;
     esac
  done
)
( cd JMTEB/src9 && python3.9 -m jmteb --embedder SentenceBertEmbedder --embedder.model_name_or_path $model --save_dir ../../result/$model )
echo '***' $model
cat result/$model/summary.json
echo ''

NVIDIA A100-SXM4-40Bだと、約5時間で以下の結果が出力された。

*** sbintuitions/modernbert-ja-130m
{
    "Classification": {
        "amazon_counterfactual_classification": {
            "macro_f1": 0.7950890220234579
        },
        "amazon_review_classification": {
            "macro_f1": 0.5487820868146802
        },
        "massive_intent_classification": {
            "macro_f1": 0.7846780827802187
        },
        "massive_scenario_classification": {
            "macro_f1": 0.8773959115883739
        }
    },
    "Reranking": {
        "esci": {
            "ndcg@10": 0.8912429272483781
        }
    },
    "Retrieval": {
        "jagovfaqs_22k": {
            "ndcg@10": 0.2023059628411459
        },
        "jaqket": {
            "ndcg@10": 0.004896766886316614
        },
        "mrtydi": {
            "ndcg@10": 0.0010101371072516164
        },
        "nlp_journal_abs_intro": {
            "ndcg@10": 0.6066659482790582
        },
        "nlp_journal_title_abs": {
            "ndcg@10": 0.04250008352802152
        },
        "nlp_journal_title_intro": {
            "ndcg@10": 0.0283856529293545
        }
    },
    "STS": {
        "jsick": {
            "spearman": 0.6866891606396426
        },
        "jsts": {
            "spearman": 0.7251711023967391
        }
    },
    "Clustering": {
        "livedoor_news": {
            "v_measure_score": 0.47841250764829696
        },
        "mewsc16": {
            "v_measure_score": 0.3174619711273742
        }
    },
    "PairClassification": {
        "paws_x_ja": {
            "binary_f1": 0.6239830208701804
        }
    }
}

Retrievalが低いのは、まあ、仕方ないだろう。あとは、まあまあの評価値かな。