LoginSignup
2
2

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?

@metaaa(めた)

[Unity] Mediapipeを使って簡易VTuberシステムを作ってみた

Posted at

はじめに

MediaPipeUnityPluginを使って簡易的なVTuberシステムを作ってみました

環境

  • Unity 2022.3.6f1
  • MediaPipeUnityPlugin v0.14.1

使用Asset

  • FinalIk(VRIK)
  • UniVRM(VRM1.0)
  • VRoidSampleModel

実装

MediaPipeUnityPluginSampleのSampleにあるFaceDetectorRunner.csを参考に改変します
以下のようにOnFaceLandmarkDetectionOutputに任意の処理を追加することでFaceLandmarkerResultを受け取りました

private void OnFaceLandmarkDetectionOutput(FaceLandmarkerResult result, Image image, long timestamp)
{
    _faceLandmarkerResultAnnotationController.DrawLater(result);

    avatarController.Track(result);
}

顔のトラッキング

MediapipeのFaceDetectorはLandmarkとblendshapeの値を出力してくれるので、この情報をもとにアバターのトラッキングを行います

顔の向きのトラッキング

顔の向きの操作はFinalIkを使用しました
ランドマークから顔の向きを計算し、結果をIkに適用しています

public void Update()
{
    _ikTargets.Head.localRotation = Quaternion.Slerp(_ikTargets.Head.localRotation, _headRot,
        _trackingParameter.ikTrackingLerp);
}

public void FaceTrack(FaceLandmarkerResult faceLandmarkResult)
{
    if (faceLandmarkResult.faceLandmarks == null) return;
    if (faceLandmarkResult.faceLandmarks.Count < _landmarkIndex) return;
    _faceLandmarks = faceLandmarkResult.faceLandmarks[_landmarkIndex].landmarks;

    _headRot = GetLookAtRotation();
}

private Quaternion GetLookAtRotation()
{
    var faceVertical = (GetFaceLandmarkPosition(LandmarkDefinition.FaceLeft) -
                        GetFaceLandmarkPosition(LandmarkDefinition.FaceRight)).normalized;
    var faceHorizontal = (GetFaceLandmarkPosition(LandmarkDefinition.FaceBottom) -
                        GetFaceLandmarkPosition(LandmarkDefinition.FaceTop)).normalized;

    var lookRollRad = Mathf.Atan2(faceHorizontal.y, faceHorizontal.x) - 90f * Mathf.Deg2Rad;
    var lookRoll = Quaternion.Euler(0, 0, lookRollRad * Mathf.Rad2Deg);
    var lookVec = Vector3.Cross(faceVertical, faceHorizontal);
    var lookQua = lookRoll * Quaternion.LookRotation(lookVec);
    return lookQua;
}

private Vector3 GetFaceLandmarkPosition(int index)
{
    var landmark = _faceLandmarks[index];
    return new Vector3(landmark.x, landmark.y, landmark.z);
}

Landmarkから大体顔の中央を通る横と縦のベクトルを求め、外積を使うことで大まかな向きを求めます

これだけではロール回転 (正面を向いたまま肩の方向に頭を倒す回転?) が考慮されないので、縦のベクトルのxyの値からAtan2を用いて角度を計算し外積で求めた値と合成しています

表情のトラッキング

表情の操作は主にmediapipeが出力するBlendShapeの値をVRM1.0のBlendShapeに適用することで実装しました
(頑張ればLandmarkから値を求めることも出来ます)

public void Update()
{
    var t = _trackingParameter.faceTrackingLerp;
    var mouseOpenWeight = Mathf.Lerp(_vrm10RuntimeExpression.GetWeight(ExpressionKey.Aa), _mouseOpenWeight, t);
    var leftBlinkWeight = Mathf.Lerp(_vrm10RuntimeExpression.GetWeight(ExpressionKey.BlinkLeft),
        _blinkWeights.leftBlink, t);
    var rightBlinkWeight = Mathf.Lerp(_vrm10RuntimeExpression.GetWeight(ExpressionKey.BlinkRight),
        _blinkWeights.rightBlink, t);
    var yaw = Mathf.Lerp(_vrm10RuntimeLookAt.Yaw, _irisYawPitch.yaw, t);
    var pitch = Mathf.Lerp(_vrm10RuntimeLookAt.Pitch, _irisYawPitch.pitch, t);

    _vrm10RuntimeExpression.SetWeight(ExpressionKey.Aa, mouseOpenWeight);
    _vrm10RuntimeExpression.SetWeight(ExpressionKey.BlinkLeft, leftBlinkWeight);
    _vrm10RuntimeExpression.SetWeight(ExpressionKey.BlinkRight, rightBlinkWeight);
    _vrm10RuntimeLookAt.SetYawPitchManually(yaw, pitch);
}

public void TrackExpressions(FaceLandmarkerResult faceLandmarkResult)
{
    if (faceLandmarkResult.faceBlendshapes == null) return;
    if (faceLandmarkResult.faceBlendshapes.Count < _landmarkIndex) return;

    var faceBlendShapes = faceLandmarkResult.faceBlendshapes[_landmarkIndex].categories;

    _mouseOpenWeight = GetMouthOpenWeight(faceBlendShapes);
    _blinkWeights = GetBlinkWeight(faceBlendShapes);
    _irisYawPitch = GetIrisYawPitch(faceBlendShapes);
}

private float GetMouthOpenWeight(List<Category> faceBlendShapes)
{
    var jawOpen = faceBlendShapes[BlendShapeDefinition.JawOpen].score;
    var mouseCurve = _trackingParameter.mouseCurve;

    return mouseCurve.Evaluate(jawOpen);
}

private (float, float) GetBlinkWeight(List<Category> faceBlendShapes)
{
    var eyeBlinkLeft = faceBlendShapes[BlendShapeDefinition.EyeBlinkLeft].score;
    var eyeBlinkRight = faceBlendShapes[BlendShapeDefinition.EyeBlinkRight].score;
    var blinkCurve = _trackingParameter.blinkCurve;

    return (blinkCurve.Evaluate(eyeBlinkLeft), blinkCurve.Evaluate(eyeBlinkRight));
}

private (float, float) GetIrisYawPitch(List<Category> faceBlendShapes)
{
    var eyeLookDownLeft = faceBlendShapes[BlendShapeDefinition.EyeLookDownLeft].score;
    var eyeLookDownRight = faceBlendShapes[BlendShapeDefinition.EyeLookDownRight].score;
    var eyeLookInLeft = faceBlendShapes[BlendShapeDefinition.EyeLookInLeft].score;
    var eyeLookInRight = faceBlendShapes[BlendShapeDefinition.EyeLookInRight].score;
    var eyeLookOutLeft = faceBlendShapes[BlendShapeDefinition.EyeLookOutLeft].score;
    var eyeLookOutRight = faceBlendShapes[BlendShapeDefinition.EyeLookOutRight].score;
    var eyeLookUpLeft = faceBlendShapes[BlendShapeDefinition.EyeLookUpLeft].score;
    var eyeLookUpRight = faceBlendShapes[BlendShapeDefinition.EyeLookUpRight].score;

    var horizontalIrisCurve = _trackingParameter.horizontalIrisCurve;
    var verticalIrisCurve = _trackingParameter.verticalIrisCurve;

    var horizontalOuter = _vrm10ObjectLookAt.HorizontalOuter;
    var horizontalInner = _vrm10ObjectLookAt.HorizontalInner;
    var verticalDown = _vrm10ObjectLookAt.VerticalDown;
    var verticalUp = _vrm10ObjectLookAt.VerticalUp;

    var lookLeft = (eyeLookInRight + eyeLookOutLeft) * 0.5f;
    var lookRight = (eyeLookInLeft + eyeLookOutRight) * 0.5f;
    var lookUp = (eyeLookUpLeft + eyeLookUpRight) * 0.5f;
    var lookDown = (eyeLookDownLeft + eyeLookDownRight) * 0.5f;

    var yaw = 0f;
    var pitch = 0f;

    var horizontalCurveXRangeDegree =
        (horizontalInner.CurveXRangeDegree + horizontalOuter.CurveXRangeDegree) * 0.5f;
    if (lookRight > lookLeft)
    {
        lookRight = horizontalIrisCurve.Evaluate(lookRight);
        yaw = Mathf.Lerp(0, horizontalCurveXRangeDegree, lookRight);
    }
    else
    {
        lookLeft = horizontalIrisCurve.Evaluate(lookLeft);
        yaw = Mathf.Lerp(0, horizontalCurveXRangeDegree * -1f, lookLeft);
    }

    if (lookUp > lookDown)
    {
        lookUp = verticalIrisCurve.Evaluate(lookUp);
        pitch = Mathf.Lerp(0, verticalUp.CurveXRangeDegree, lookUp);
    }
    else
    {
        lookDown = verticalIrisCurve.Evaluate(lookDown);
        pitch = Mathf.Lerp(0, verticalDown.CurveXRangeDegree * -1f, lookDown);
    }

    return (yaw, pitch);
}

瞬きと口の開き具合

Mediapipeが出力したBlendShapeの値をAnimationCurveで調整して適用しています(簡単)
VRMには各発音?の口の形のBlendShapeがあるので頑張って値を求めてみるのもよいかもしれません

private float GetMouthOpenWeight(List<Category> faceBlendShapes)
{
    var jawOpen = faceBlendShapes[BlendShapeDefinition.JawOpen].score;
    var mouseCurve = _trackingParameter.mouseCurve;

    return mouseCurve.Evaluate(jawOpen);
}

private (float, float) GetBlinkWeight(List<Category> faceBlendShapes)
{
    var eyeBlinkLeft = faceBlendShapes[BlendShapeDefinition.EyeBlinkLeft].score;
    var eyeBlinkRight = faceBlendShapes[BlendShapeDefinition.EyeBlinkRight].score;
    var blinkCurve = _trackingParameter.blinkCurve;

    return (blinkCurve.Evaluate(eyeBlinkLeft), blinkCurve.Evaluate(eyeBlinkRight));
}

瞳のトラッキング

VRM1.0には瞳を動かすためだと思われるBlendShapeが存在していますが、SampleModelだと動作しなかったのでSetYawPitchManuallyを使用しました
MediapipeのBlendShapeは各瞳毎の値を出力しますが、SetYawPitchManuallyだと両目とも同じ値になるので以下の処理でいい感じの値を求めました

private (float, float) GetIrisYawPitch(List<Category> faceBlendShapes)
{
    var eyeLookDownLeft = faceBlendShapes[BlendShapeDefinition.EyeLookDownLeft].score;
    var eyeLookDownRight = faceBlendShapes[BlendShapeDefinition.EyeLookDownRight].score;
    var eyeLookInLeft = faceBlendShapes[BlendShapeDefinition.EyeLookInLeft].score;
    var eyeLookInRight = faceBlendShapes[BlendShapeDefinition.EyeLookInRight].score;
    var eyeLookOutLeft = faceBlendShapes[BlendShapeDefinition.EyeLookOutLeft].score;
    var eyeLookOutRight = faceBlendShapes[BlendShapeDefinition.EyeLookOutRight].score;
    var eyeLookUpLeft = faceBlendShapes[BlendShapeDefinition.EyeLookUpLeft].score;
    var eyeLookUpRight = faceBlendShapes[BlendShapeDefinition.EyeLookUpRight].score;

    var horizontalIrisCurve = _trackingParameter.horizontalIrisCurve;
    var verticalIrisCurve = _trackingParameter.verticalIrisCurve;

    var horizontalOuter = _vrm10ObjectLookAt.HorizontalOuter;
    var horizontalInner = _vrm10ObjectLookAt.HorizontalInner;
    var verticalDown = _vrm10ObjectLookAt.VerticalDown;
    var verticalUp = _vrm10ObjectLookAt.VerticalUp;

    var lookLeft = (eyeLookInRight + eyeLookOutLeft) * 0.5f;
    var lookRight = (eyeLookInLeft + eyeLookOutRight) * 0.5f;
    var lookUp = (eyeLookUpLeft + eyeLookUpRight) * 0.5f;
    var lookDown = (eyeLookDownLeft + eyeLookDownRight) * 0.5f;

    var yaw = 0f;
    var pitch = 0f;

    var horizontalCurveXRangeDegree =
        (horizontalInner.CurveXRangeDegree + horizontalOuter.CurveXRangeDegree) * 0.5f;
    if (lookRight > lookLeft)
    {
        lookRight = horizontalIrisCurve.Evaluate(lookRight);
        yaw = Mathf.Lerp(0, horizontalCurveXRangeDegree, lookRight);
    }
    else
    {
        lookLeft = horizontalIrisCurve.Evaluate(lookLeft);
        yaw = Mathf.Lerp(0, horizontalCurveXRangeDegree * -1f, lookLeft);
    }

    if (lookUp > lookDown)
    {
        lookUp = verticalIrisCurve.Evaluate(lookUp);
        pitch = Mathf.Lerp(0, verticalUp.CurveXRangeDegree, lookUp);
    }
    else
    {
        lookDown = verticalIrisCurve.Evaluate(lookDown);
        pitch = Mathf.Lerp(0, verticalDown.CurveXRangeDegree * -1f, lookDown);
    }

    return (yaw, pitch);
}

参考

2
2
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
2
2

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?