NLP4J - Java で 英語の形態素解析（Stanford NLPを利用）

Index

Stanford NLPとは

スタンフォードNLPグループが提供する、主要な計算言語学の問題に対して、統計NLP、深層学習NLP、およびルールベースのNLPツールです。

Software - The Stanford Natural Language Processing Group
https://nlp.stanford.edu/software/

項目	説明
提供者	スタンフォード大学 スタンフォードNLPグループ
提供形式	Java ライブラリ

Example (without NLP4J)

package hello.stanford;

import java.util.List;
import java.util.Properties;
import edu.stanford.nlp.ling.CoreAnnotations.CharacterOffsetBeginAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.CharacterOffsetEndAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.LemmaAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.SentenceIndexAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.SentencesAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokenBeginAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokenEndAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.util.CoreMap;

public class HelloStanfordNlp001PosLemma {
    public static void main(String[] args) {
        String text = "This is test. He runs fast.";
        Properties properties = new Properties();
        // "tokenize, ssplit, pos, lemma, depparse"
        // https://stanfordnlp.github.io/CoreNLP/annotators.html
        // tokenize : TokenizerAnnotator
        // ssplit : WordsToSentencesAnnotator : Splits a sequence of tokens into
        // sentences.
        // pos : POSTaggerAnnotator : Labels tokens with their POS tag.
        // lemma : 見出し
        // lemma : MorphaAnnotator : Generates the word lemmas for all tokens in the
        // corpus.
        properties.setProperty("annotators", "tokenize, ssplit, pos, lemma");
        StanfordCoreNLP coreNLP = new StanfordCoreNLP(properties);
        Annotation annotation = new Annotation(text);
        coreNLP.annotate(annotation);
        {
            List<CoreLabel> cl = annotation.get(TokensAnnotation.class);
            for (CoreLabel label : cl) {
                System.err.println("<token>");
                // [class edu.stanford.nlp.ling.CoreAnnotations$ValueAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$TextAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$OriginalTextAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$CharacterOffsetBeginAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$CharacterOffsetEndAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$BeforeAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$AfterAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$IsNewlineAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$TokenBeginAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$TokenEndAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$IndexAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$SentenceIndexAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$PartOfSpeechAnnotation,
                // class edu.stanford.nlp.ling.CoreAnnotations$LemmaAnnotation]
                System.err.println("SentenceIndexAnnotation=" + label.get(SentenceIndexAnnotation.class));
                System.err.println("PartOfSpeechAnnotation=" + label.get(PartOfSpeechAnnotation.class));
                System.err.println("LemmaAnnotation=" + label.get(LemmaAnnotation.class));
                System.err.println("</token>");
            }
        }
        System.err.println("---");
        System.err.println("TextAnnotation");
        // class edu.stanford.nlp.ling.CoreAnnotations$TextAnnotation,
        {
            System.err.println(annotation.get(TextAnnotation.class));
        }
        // class edu.stanford.nlp.ling.CoreAnnotations$SentencesAnnotation
        System.err.println("SentencesAnnotation");

        {
            List<CoreMap> sentenceMap = annotation.get(SentencesAnnotation.class);

            for (CoreMap label : sentenceMap) {
                System.err.println("<sentence>");
                // [class edu.stanford.nlp.ling.CoreAnnotations$TextAnnotation,
                System.err.println("TextAnnotation=" + label.get(TextAnnotation.class));
                // class edu.stanford.nlp.ling.CoreAnnotations$CharacterOffsetBeginAnnotation,
                System.err.println(
                            "CharacterOffsetBeginAnnotation=" + label.get(CharacterOffsetBeginAnnotation.class));
                // class edu.stanford.nlp.ling.CoreAnnotations$CharacterOffsetEndAnnotation,
                System.err.println("CharacterOffsetEndAnnotation=" + label.get(CharacterOffsetEndAnnotation.class));
                // class edu.stanford.nlp.ling.CoreAnnotations$TokensAnnotation,
                System.err.println("TokensAnnotation=" + label.get(TokensAnnotation.class));
                // class edu.stanford.nlp.ling.CoreAnnotations$SentenceIndexAnnotation,
                System.err.println("SentenceIndexAnnotation=" + label.get(SentenceIndexAnnotation.class));
                // class edu.stanford.nlp.ling.CoreAnnotations$TokenBeginAnnotation,
                System.err.println("TokenBeginAnnotation=" + label.get(TokenBeginAnnotation.class));
                // class edu.stanford.nlp.ling.CoreAnnotations$TokenEndAnnotation]
                System.err.println("TokenEndAnnotation=" + label.get(TokenEndAnnotation.class));
                System.err.println("</sentence>");
            }
        }
    }
}

Output Example (without NLP4J)

<token>
SentenceIndexAnnotation=0
PartOfSpeechAnnotation=DT
LemmaAnnotation=this
</token>
<token>
SentenceIndexAnnotation=0
PartOfSpeechAnnotation=VBZ
LemmaAnnotation=be
</token>
<token>
SentenceIndexAnnotation=0
PartOfSpeechAnnotation=NN
LemmaAnnotation=test
</token>
<token>
SentenceIndexAnnotation=0
PartOfSpeechAnnotation=.
LemmaAnnotation=.
</token>
<token>
SentenceIndexAnnotation=1
PartOfSpeechAnnotation=PRP
LemmaAnnotation=he
</token>
<token>
SentenceIndexAnnotation=1
PartOfSpeechAnnotation=VBZ
LemmaAnnotation=run
</token>
<token>
SentenceIndexAnnotation=1
PartOfSpeechAnnotation=RB
LemmaAnnotation=fast
</token>
<token>
SentenceIndexAnnotation=1
PartOfSpeechAnnotation=.
LemmaAnnotation=.
</token>
---
TextAnnotation
This is test. He runs fast.
SentencesAnnotation
<sentence>
TextAnnotation=This is test.
CharacterOffsetBeginAnnotation=0
CharacterOffsetEndAnnotation=13
TokensAnnotation=[This-1, is-2, test-3, .-4]
SentenceIndexAnnotation=0
TokenBeginAnnotation=0
TokenEndAnnotation=4
</sentence>
<sentence>
TextAnnotation=He runs fast.
CharacterOffsetBeginAnnotation=14
CharacterOffsetEndAnnotation=27
TokensAnnotation=[He-1, runs-2, fast-3, .-4]
SentenceIndexAnnotation=1
TokenBeginAnnotation=4
TokenEndAnnotation=8
</sentence>

Maven Dependencies for NLP4J

<!-- https://mvnrepository.com/artifact/org.nlp4j/nlp4j-stanford -->
<dependency>
    <groupId>org.nlp4j</groupId>
    <artifactId>nlp4j-core</artifactId>
    <version>[1.3.1.0,)</version>
</dependency>
<dependency>
    <groupId>org.nlp4j</groupId>
    <artifactId>nlp4j-stanford</artifactId>
    <version>[1.3.0.0,)</version>
</dependency>
<!-- https://mvnrepository.com/artifact/edu.stanford.nlp/stanford-corenlp -->
<dependency>
    <groupId>edu.stanford.nlp</groupId>
    <artifactId>stanford-corenlp</artifactId>
    <version>4.0.0</version>
    <scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/edu.stanford.nlp/stanford-corenlp -->
<dependency>
    <groupId>edu.stanford.nlp</groupId>
    <artifactId>stanford-corenlp</artifactId>
    <version>4.0.0</version>
    <classifier>models</classifier>
    <scope>provided</scope>
</dependency>

コード with NLP4J

package nlp4j.stanford.examples;

import nlp4j.Document;
import nlp4j.Keyword;
import nlp4j.impl.DefaultDocument;
import nlp4j.stanford.StanfordPosAnnotator;

public class StanfordPosAnnotatorExample0 {

    public static void main(String[] args) throws Exception {
        Document doc = new DefaultDocument();
        {
            doc.putAttribute("text", "I eat sushi with chopsticks.");
        }
        StanfordPosAnnotator ann = new StanfordPosAnnotator();
        {
            ann.setProperty("target", "text");
        }
        ann.annotate(doc); // do annotation
        for (Keyword kwd : doc.getKeywords()) {
            System.err.println(kwd);
        }
    }
}

結果 with NLP4J

I [facet=word.PRP, str=I]
eat [facet=word.VBP, str=eat]
sushi [facet=word.NN, str=sushi]
with [facet=word.IN, str=with]
chopstick [facet=word.NNS, str=chopsticks]
. [facet=word.., str=.]

Index