JavaでXML署名を実装する必要があったのでまとめておく。
基本的には以下のコードをJava 11で実際に動くように少しいじっただけ。

環境

macOS 10.13.6 (High Sierra)
OpenSSL 1.0.2p
Java 11.0.1 (OpenJDK)

準備：秘密鍵・証明書の作成

RSA秘密鍵・X.509公開鍵証明書のペアを作成
$ openssl req -out alice.crt -nodes -keyout alice.pem -newkey rsa:4096 -sha256 -x509 -days 365

証明書の確認

$ openssl x509 -in alice.crt -noout -text

Javaで秘密鍵を読み込むには、秘密鍵をPKCS #8・DER形式に変換する必要がある。
$ openssl pkcs8 -in alice.pem -outform DER -out alice.pk8 -topk8 -nocrypt

ファイルの状況は次の通り。

alice.pk8：PKCS #8・DER形式の秘密鍵（RSA 4096 bit）
alice.crt：X.509公開鍵証明書（RSA・SHA-256）
alice.pem：PEM形式の秘密鍵（今回は使わない）

サンプルXMLファイル

envelope.xml

<!-- ref. https://docs.oracle.com/javase/6/docs/technotes/guides/security/xmldsig/envelope.xml -->
<?xml version="1.0" encoding="UTF-8"?>
<Envelope xmlns="urn:envelope">
</Envelope>

XML署名プログラム

GenEnveloped.java

// ref. https://docs.oracle.com/javase/6/docs/technotes/guides/security/xmldsig/GenEnveloped.java

import javax.xml.crypto.dsig.*;
import javax.xml.crypto.dsig.dom.DOMSignContext;
import javax.xml.crypto.dsig.keyinfo.*;
import javax.xml.crypto.dsig.spec.*;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.security.*;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.interfaces.RSAPrivateKey;
import java.security.spec.PKCS8EncodedKeySpec;
import java.util.Collections;
import java.util.List;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.*;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.w3c.dom.Document;

public class GenEnveloped {
  public static void main(String[] args) {
    if (args.length != 4) {
      System.err.println("Usage: java GenEnveloped [input XML path] [output XML path] [private key path (pk8)] [certificate path]");
      System.exit(1);
    }

    try {
      genEnveloped(args[0], args[1], args[2], args[3]);
    } catch (Exception e) {
      System.err.println(e);
      System.exit(1);
    }
  }

  public static void genEnveloped(String inXmlPath, String outXmlPath, String privateKeyPath, String certPath) throws Exception {
    // Create a DOM XMLSignatureFactory that will be used to generate the
    // enveloped signature
    XMLSignatureFactory fac = XMLSignatureFactory.getInstance("DOM");

    // Create a Reference to the enveloped document (in this case we are
    // signing the whole document, so a URI of "" signifies that) and
    // also specify the SHA256 digest algorithm and the ENVELOPED Transform.
    DigestMethod dm = fac.newDigestMethod(DigestMethod.SHA256, null);
    List<Transform> transforms = Collections.singletonList(fac.newTransform(Transform.ENVELOPED, (TransformParameterSpec) null));
    Reference ref = fac.newReference("", dm, transforms, null, null);

    // Create the SignedInfo
    CanonicalizationMethod cm = fac.newCanonicalizationMethod(CanonicalizationMethod.INCLUSIVE_WITH_COMMENTS, (C14NMethodParameterSpec) null);
    SignatureMethod sm = fac.newSignatureMethod(SignatureMethod.RSA_SHA256, null);
    List<Reference> references = Collections.singletonList(ref);
    SignedInfo si = fac.newSignedInfo(cm, sm, references);

    // Read a RSA private key
    FileInputStream fis = new FileInputStream(privateKeyPath);
    byte[] privateKeyByte = new byte[fis.available()];
    fis.read(privateKeyByte);
    fis.close();
    PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(privateKeyByte);
    KeyFactory kf = KeyFactory.getInstance("RSA");
    RSAPrivateKey privateKey = (RSAPrivateKey) kf.generatePrivate(keySpec);

    // Read a X.509 certificate
    KeyInfoFactory kif = fac.getKeyInfoFactory();
    CertificateFactory cf = CertificateFactory.getInstance("X.509");
    X509Certificate cert = (X509Certificate) cf.generateCertificate(new FileInputStream(certPath));
    X509Data x509Data = kif.newX509Data(Collections.singletonList(cert));

    // Create a KeyInfo and add the X509Data to it
    KeyInfo ki = kif.newKeyInfo(Collections.singletonList(x509Data));

    // Instantiate the document to be signed
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    dbf.setNamespaceAware(true);
    Document doc = dbf.newDocumentBuilder().parse(new FileInputStream(inXmlPath));

    // Create a DOMSignContext and specify the RSA PrivateKey and
    // location of the resulting XMLSignature's parent element
    DOMSignContext dsc = new DOMSignContext(privateKey, doc.getDocumentElement());

    // Create the XMLSignature (but don't sign it yet)
    XMLSignature signature = fac.newXMLSignature(si, ki);

    // Marshal, generate (and sign) the enveloped signature
    signature.sign(dsc);

    // output the resulting document
    OutputStream os = new FileOutputStream(outXmlPath);
    TransformerFactory tf = TransformerFactory.newInstance();
    Transformer trans = tf.newTransformer();
    trans.transform(new DOMSource(doc), new StreamResult(os));
  }
}

コンパイル
$ javac GenEnveloped.java

Usage
$ java GenEnveloped [input XML path] [output XML path] [private key path (pk8)] [certificate path]

実行例
$ java GenEnveloped envelope.xml envelopedSignature.xml alice.pk8 alice.crt

XML署名検証プログラム

Validate.java

// ref. https://docs.oracle.com/javase/6/docs/technotes/guides/security/xmldsig/Validate.java

import javax.xml.crypto.*;
import javax.xml.crypto.dsig.*;
import javax.xml.crypto.dsig.dom.DOMValidateContext;
import javax.xml.crypto.dsig.keyinfo.*;
import java.io.FileInputStream;
import java.security.*;
import java.security.cert.X509Certificate;
import java.util.Iterator;
import javax.xml.parsers.DocumentBuilderFactory;

import org.w3c.dom.Document;
import org.w3c.dom.NodeList;

public class Validate {
  public static void main(String[] args) {
    if (args.length != 1) {
      System.err.println("Usage: java Validate [input XML path]");
      System.exit(1);
    }

    try {
      validate(args[0]);
    } catch (Exception e) {
      System.err.println(e);
      System.exit(1);
    }
  }

  public static boolean validate(String inXmlPath) throws Exception {
    // Instantiate the document to be validated
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    dbf.setNamespaceAware(true);
    Document doc = dbf.newDocumentBuilder().parse(new FileInputStream(inXmlPath));

    // Find Signature element
    NodeList nl = doc.getElementsByTagNameNS(XMLSignature.XMLNS, "Signature");
    if (nl.getLength() == 0)
      throw new Exception("Cannot find Signature element");

    // Create a DOM XMLSignatureFactory that will be used to unmarshal the
    // document containing the XMLSignature
    XMLSignatureFactory fac = XMLSignatureFactory.getInstance("DOM");

    // Create a DOMValidateContext and specify a KeyValue KeySelector
    // and document context
    DOMValidateContext valContext = new DOMValidateContext(new KeyValueKeySelector(), nl.item(0));

    // unmarshal the XMLSignature
    XMLSignature signature = fac.unmarshalXMLSignature(valContext);

    // Validate the XMLSignature (generated above)
    boolean coreValidity = signature.validate(valContext);

    // Check core validation status
    if (!coreValidity) {
      System.err.println("Signature failed core validation");
      boolean sv = signature.getSignatureValue().validate(valContext);
      System.out.println("signature validation status: " + sv);
      // check the validation status of each Reference
      Iterator i = signature.getSignedInfo().getReferences().iterator();
      for (int j = 0; i.hasNext(); j++) {
        boolean refValid = ((Reference) i.next()).validate(valContext);
        System.out.println("ref[" + j + "] validity status: " + refValid);
      }
    } else
      System.out.println("Signature passed core validation");
    return coreValidity;
  }

  /**
   * KeySelector which retrieves the public key out of the
   * KeyValue element and returns it.
   * NOTE: If the key algorithm doesn't match signature algorithm,
   * then the public key will be ignored.
   */
  private static class KeyValueKeySelector extends KeySelector {
    public KeySelectorResult select(KeyInfo keyInfo, KeySelector.Purpose purpose, AlgorithmMethod method, XMLCryptoContext context) throws KeySelectorException {
      if (keyInfo == null)
        throw new KeySelectorException("Null KeyInfo object!");
      SignatureMethod sm = (SignatureMethod) method;

      for (Object keyInfoContent : keyInfo.getContent()) {
        if (keyInfoContent instanceof X509Data) {
          for (Object x509Content : ((X509Data) keyInfoContent).getContent()) {
            X509Certificate cert = (X509Certificate) x509Content;
            PublicKey pk = cert.getPublicKey();
            // make sure algorithm is compatible with method
            if (algEquals(sm.getAlgorithm(), pk.getAlgorithm()))
              return new SimpleKeySelectorResult(pk);
          }
        }
      }
      throw new KeySelectorException("No KeyValue element found!");
    }

    static boolean algEquals(String algURI, String algName) {
      if (algName.equalsIgnoreCase("RSA") && algURI.equalsIgnoreCase(SignatureMethod.RSA_SHA256))
        return true;
      else if (algName.equalsIgnoreCase("DSA") && algURI.equalsIgnoreCase(SignatureMethod.DSA_SHA1))
        return true;
      else if (algName.equalsIgnoreCase("RSA") && algURI.equalsIgnoreCase(SignatureMethod.RSA_SHA1))
        return true;
      else
        return false;
    }
  }

  private static class SimpleKeySelectorResult implements KeySelectorResult {
    private PublicKey pk;

    SimpleKeySelectorResult(PublicKey pk) {
      this.pk = pk;
    }

    public Key getKey() {
      return pk;
    }
  }
}

コンパイル
$ javac Validate.java

Usage
$ java Validate [input XML path]

実行例（検証成功）

$ java Validate envelopedSignature.xml
Signature passed core validation

実行例（検証失敗）

$ java Validate envelopedSignature.xml
Signature failed core validation
signature validation status: false
ref[0] validity status: true

P.S.

ソースコードはこちら。
haru52/xmldsig

References