LoginSignup
1
2

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?

More than 1 year has passed since last update.

@KayaTakashiro(Kaya Takashiro)

Oculus の指の動きを別の3Dモデルの指に反映させる方法!

Last updated at Posted at 2022-04-23

目的

OculusのHand Trackingによる指の動きを3Dモデルの指の動きに反映させる

デバックするとおかしくなる時がかなりあるのでこちらを使ってください

VRMのエージェントの指の動きをOculus の手の動きでミラーリングする方法

以下、OculusQuest2で動作させる前はうまく動きます。

3Dモデルと同じBoneの構造をもったhandを用意する！

キャプチャ.PNG

！！！Unity Rotation Fixで、オブジェクトの向きをUnity仕様に直しておく。

3DモデルをFBX形式で出力！ > HandLeft.fbx, HandRight.fbxとする。

キャプチャ.PNG

VRM形式と同じモデルを用いた、FBX形式の3Dモデルを出力する。

FBXのモデルとVRMのモデルをUnityのプロジェクトにImportし、FBXのモデルにVRMのマテリアルを設定する。

VRM形式のモデルは使用しないため、SceneではInactiveに設定しておきます。

FBX形式のモデルとVRM形式のモデルではlocalRotationが異なるために、こんなめんどくさいことをしています。

HandLeft.fbx, HandRight.fbxをImportする。

  1. 一番上の親に OVRHand と OVRCustomSkeletonを設定する

  2. Skeleton Type をそれぞれHand LeftとHand Rightに設定する。

Oculus Pluginの改造！

1. OVRCustomSkeletonについて
OVRCustomSkeleton.cs
	private static readonly string[] _fbxHandSidePrefix = { "L_", "R_" };
	private static readonly string _fbxHandBonePrefix = "J_Bip_";

	private static readonly string[] _fbxHandBoneNames =
	{
		"Hand",
		"Thumb1",
		"Thumb2",
		"Thumb3",
		"Index1",
		"Index2",
		"Index3",
		"Middle1",
		"Middle2",
		"Middle3",
		"Ring1",
		"Ring2",
		"Ring3",
		"Little1",
		"Little2",
		"Little3"
	};

	private static readonly string[] _fbxHandFingerNames =
	{
		"Thumb",
		"Index",
		"Middle",
		"Ring",
		"Little"
	};
2. OVRSkeleton.csについて
OVRSkeleton.cs
	public enum BoneId
	{
		Invalid                 = OVRPlugin.BoneId.Invalid,

		// hand bones
		Hand_Start              = OVRPlugin.BoneId.Hand_Start,
		Hand_Hand               = OVRPlugin.BoneId.Hand_Hand,          // root frame of the hand, where the wrist is located
		Hand_Thumb1             = OVRPlugin.BoneId.Hand_Thumb1,             // thumb metacarpal bone
		Hand_Thumb2             = OVRPlugin.BoneId.Hand_Thumb2,             // thumb proximal phalange bone
		Hand_Thumb3             = OVRPlugin.BoneId.Hand_Thumb3,             // thumb distal phalange bone
		Hand_Index1             = OVRPlugin.BoneId.Hand_Index1,             // index proximal phalange bone
		Hand_Index2             = OVRPlugin.BoneId.Hand_Index2,             // index intermediate phalange bone
		Hand_Index3             = OVRPlugin.BoneId.Hand_Index3,             // index distal phalange bone
		Hand_Middle1            = OVRPlugin.BoneId.Hand_Middle1,            // middle proximal phalange bone
		Hand_Middle2            = OVRPlugin.BoneId.Hand_Middle2,            // middle intermediate phalange bone
		Hand_Middle3            = OVRPlugin.BoneId.Hand_Middle3,            // middle distal phalange bone
		Hand_Ring1              = OVRPlugin.BoneId.Hand_Ring1,              // ring proximal phalange bone
		Hand_Ring2              = OVRPlugin.BoneId.Hand_Ring2,              // ring intermediate phalange bone
		Hand_Ring3              = OVRPlugin.BoneId.Hand_Ring3,              // ring distal phalange bone
		Hand_Little1             = OVRPlugin.BoneId.Hand_Little1,             // pinky proximal phalange bone
		Hand_Little2             = OVRPlugin.BoneId.Hand_Little2,             // pinky intermediate phalange bone
		Hand_Little3             = OVRPlugin.BoneId.Hand_Little3,             // pinky distal phalange bone
		Hand_MaxSkinnable       = OVRPlugin.BoneId.Hand_MaxSkinnable,
		// Bone tips are position only. They are not used for skinning but are useful for hit-testing.
		// NOTE: Hand_ThumbTip == Hand_MaxSkinnable since the extended tips need to be contiguous
		Hand_End                = OVRPlugin.BoneId.Hand_End,


		// add new bones here

		Max                     = OVRPlugin.BoneId.Max
	}

まず、このようにVRMのモデルの構造に合わせて書き替える

  • OVRSkeleton.BoneId.Hand_WriteRoot > OVRSkeleton.BoneId.Hand_Hand に書き換える

  • Hand_Pinky1~3はHand_Little1~3に書き換える

  • それ以外は削除

3. OVRPlugin.csを書き換える
OVRPlugin.cs
public enum BoneId
	{
		Invalid                 = -1,

		// hand bones
		Hand_Start              = 0,
		Hand_Hand          		= Hand_Start + 0, // root frame of the hand, where the wrist is located
		Hand_Thumb1             = Hand_Start + 1, // thumb metacarpal bone
		Hand_Thumb2             = Hand_Start + 2, // thumb proximal phalange bone
		Hand_Thumb3             = Hand_Start + 3, // thumb distal phalange bone
		Hand_Index1             = Hand_Start + 4, // index proximal phalange bone
		Hand_Index2             = Hand_Start + 5, // index intermediate phalange bone
		Hand_Index3             = Hand_Start + 6, // index distal phalange bone
		Hand_Middle1            = Hand_Start + 7, // middle proximal phalange bone
		Hand_Middle2            = Hand_Start + 8, // middle intermediate phalange bone
		Hand_Middle3            = Hand_Start + 9, // middle distal phalange bone
		Hand_Ring1              = Hand_Start + 10, // ring proximal phalange bone
		Hand_Ring2              = Hand_Start + 11, // ring intermediate phalange bone
		Hand_Ring3              = Hand_Start + 12, // ring distal phalange bone
		Hand_Little1             = Hand_Start + 13, // pinky proximal phalange bone
		Hand_Little2             = Hand_Start + 14, // pinky intermediate phalange bone
		Hand_Little3             = Hand_Start + 15, // pinky distal phalange bone
		Hand_MaxSkinnable       = Hand_Start + 16,
		// Bone tips are position only. They are not used for skinning but are useful for hit-testing.
		// NOTE: Hand_ThumbTip == Hand_MaxSkinnable since the extended tips need to be contiguous
		Hand_End                = Hand_MaxSkinnable + 0,


		// add new bones here

		Max = ((int)Hand_End > 50) ? (int)Hand_End : 50,
	}

	public enum HandFinger
	{
		Thumb = 0,
		Index = 1,
		Middle = 2,
		Ring = 3,
		Little = 4,
		Max = 5,
	}

	[Flags]
	public enum HandFingerPinch
	{
		Thumb  = (1 << HandFinger.Thumb),
		Index  = (1 << HandFinger.Index),
		Middle = (1 << HandFinger.Middle),
		Ring   = (1 << HandFinger.Ring),
		Little  = (1 << HandFinger.Little),
	}
4. OVRHand.cs　のPinkyをLittleに変換する。
5. OVRTrackedKeyboardHands.csについて

  1. PinkyをLittleに変換する

  2. b_l_をJ_Bip_Lに、b_r_をJ_Bip_Rに変換する

  3. 指の名前の頭文字を大文字にする。

  4. Hand_WristRootを Hand_Handに変換する

6.OVRCustomSkeleton.csについて
OVRCustomSkeleton.cs

	private static string FbxBoneNameFromBoneId(SkeletonType skeletonType, BoneId bi)
	{
		{
			return _fbxHandBonePrefix + _fbxHandSidePrefix[(int)skeletonType] + _fbxHandBoneNames[(int)bi];
		}
	}

のように書き換える。

7.OVRSkeletonData.csについて

  1. Hand_ForearmStabについてはHand_Handに書き換える

  2. Hand_Pinky1~3はHand_Little1~3に書き換える

  3. 他は削除

8.FingerTipPokeTool.csについて

PinkyをLittleに変換

これでエラー処理は全部完了！！！

OVRCustomSkeleton.csのAuto Map Bonesで正常にMappingされるかを確認する

エージェントとアバターの指の動きの同期のスクリプト！

便宜上、OVRHandをアッタッチしたHandをアバター、FBXでインポートしたモデルをエージェントとします。

  1. Hierachyに空のオブジェクトを設置して、名前を"Tracer"にする

  2. Tracerの子オブジェクトとして、"LeftHandTracer"と"RightHandTracer"を用意する。

  3. 以下の GetVRMHand.cs, HandController.cs, HandType.cs, VRMAgentHand.cs, VRMAvatarHand.csの5つのスクリプトを用意する。

GetVRMHand.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class GetVRMHand : MonoBehaviour
{
//Wristの親をHandRootに設定する必要がある。

    [SerializeField] Transform VRMHandRoot;
	private List<Transform> VRMHandBones = new List<Transform>(new Transform[(int)VRMBoneId.Hand_End]);

    public enum VRMBoneId
	{
        Hand_Start              = 0,
		Hand_Hand		        = Hand_Start + 0, // root frame of the hand, where the wrist is located
		Hand_Thumb1             = Hand_Start + 1, // thumb metacarpal bone
		Hand_Thumb2             = Hand_Start + 2, // thumb proximal phalange bone
		Hand_Thumb3             = Hand_Start + 3, // thumb distal phalange bone
		Hand_Index1             = Hand_Start + 4, // index proximal phalange bone
		Hand_Index2             = Hand_Start + 5, // index intermediate phalange bone
		Hand_Index3             = Hand_Start + 6, // index distal phalange bone
		Hand_Middle1            = Hand_Start + 7, // middle proximal phalange bone
		Hand_Middle2            = Hand_Start + 8, // middle intermediate phalange bone
		Hand_Middle3            = Hand_Start + 9, // middle distal phalange bone
		Hand_Ring1              = Hand_Start + 10, // ring proximal phalange bone
		Hand_Ring2              = Hand_Start + 11, // ring intermediate phalange bone
		Hand_Ring3              = Hand_Start + 12, // ring distal phalange bone
		Hand_Little1             = Hand_Start + 13, // pinky proximal phalange bone
		Hand_Little2             = Hand_Start + 14, // pinky intermediate phalange bone
		Hand_Little3             = Hand_Start + 15, // pinky distal phalange bone
		Hand_MaxSkinnable       = Hand_Start + 16,
		// Bone tips are position only. They are not used for skinning but are useful for hit-testing.
		// NOTE: Hand_ThumbTip == Hand_MaxSkinnable since the extended tips need to be contiguous
		Hand_End                = Hand_MaxSkinnable + 0,
		Invalid                 = -1,
		// hand bones
		Max                     = ((int)Hand_End > 50) ? (int)Hand_End : 50,
	}

    public enum Hand
    {
        None = -1,
        LeftHand = 0,
        Righthand = 1
    }

	private static readonly string[] _fbxHandSidePrefix = { "L_", "R_" };
	private static readonly string _fbxHandBonePrefix = "J_Bip_";
    
	private static readonly string[] _fbxHandBoneNames =
	{
		"Hand",
		"Thumb1",
		"Thumb2",
		"Thumb3",
		"Index1",
		"Index2",
		"Index3",
		"Middle1",
		"Middle2",
		"Middle3",
		"Ring1",
		"Ring2",
		"Ring3",
		"Little1",
		"Little2",
		"Little3"
	};
	public VRMBoneId GetCurrentStartBoneId()
	{
        return VRMBoneId.Hand_Start;
	}
	public VRMBoneId GetCurrentEndBoneId()
	{
        return VRMBoneId.Hand_End;
	}
    	private static string FbxBoneNameFromBoneId(HandType.Hand handType, VRMBoneId bi)
	{
		{
		    return _fbxHandBonePrefix + _fbxHandSidePrefix[(int)handType] + _fbxHandBoneNames[(int)bi];
		}
	}
    
    public Transform[] GetHand(HandType.Hand handType){
		VRMBoneId start = GetCurrentStartBoneId();
		VRMBoneId end = GetCurrentEndBoneId();
        if (start != VRMBoneId.Invalid && end != VRMBoneId.Invalid)
		{
			for (int bi = (int)start; bi < (int)end; ++bi)
			{
				string fbxBoneName = FbxBoneNameFromBoneId(handType, (VRMBoneId)bi);
				Transform t = VRMHandRoot.FindChildRecursive(fbxBoneName); 
				if (t != null)
				{

					VRMHandBones[(int)bi] = t;
				}
			}
		}

        return VRMHandBones.ToArray();
    }
}
HandController.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class HandController : MonoBehaviour
{

    [SerializeField]
    private HandType.Hand HandType;

    private Transform[] AgentBones;
    private Transform[] AvatarBones;

    private Quaternion[] preAvatarlocalRotations = new Quaternion[(int)VRMAvatarHand.VRMBoneId.Hand_End];


    // Start is called before the first frame update
    void Start()
    {
        var AgentHand = gameObject.GetComponent<VRMAgentHand>();
        AgentBones = AgentHand.GetHand(HandType);

        var AvatarHand = gameObject.GetComponent<VRMAvatarHand>();
        AvatarBones = AvatarHand.GetHand(HandType);

        if (AgentBones.Length != AvatarBones.Length){
            Debug.Log("Length is not equal with Agent hands and Avatar hands ->" + AgentBones.Length + " vs " + AvatarBones.Length);
        }
        for(int id = 1; id  < AgentBones.Length; id++){
            preAvatarlocalRotations[id] = AvatarBones[id].localRotation;
        }        
    }

    // Update is called once per frame
    void Update()
    {
     for(int id = 1; id  < AvatarBones.Length; id++){
        var timeDiffAvatarRotation = preAvatarlocalRotations[id] * Quaternion.Inverse(AvatarBones[id].localRotation);
        AgentBones[id].localRotation = AgentBones[id].localRotation * Quaternion.Inverse(timeDiffAvatarRotation);
        preAvatarlocalRotations[id] = AvatarBones[id].localRotation;
        }   
    }
}
HandType.cs
using System.Collections;
using System.Collections.Generic;
using System;
using UnityEngine;

public class HandType : MonoBehaviour
{
    public enum Hand
    {
        None = -1,
        LeftHand = 0,
        Righthand = 1
    }
    public Transform[] getAllTransform(Transform parent){
        // 親を含む子オブジェクトを再帰的に取得
        var parentAndChildren = parent.GetComponentsInChildren<Transform>();
        // 子オブジェクトの返却用配列作成
        var children = new Transform[parentAndChildren.Length - 1];

        // 親を除く子オブジェクトを結果にコピー
        Array.Copy(parentAndChildren, 1, children, 0, children.Length);

        // 子オブジェクトが再帰的に格納された配列
        return children;
    }
}
VRMAgentHand.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class VRMAgentHand : GetVRMHand
{
}
VRMAvatarHand.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class VRMAvatarHand : GetVRMHand
{
}

  1. 5つのスクリプトのうち、GetVRMHand.cs以外の4つのスクリプトをLeftHandTracerとRightHandTracerにアタッチする

  2. VRMAgentHandには FBXモデルの　J_Bip_L_LowerArmかJ_Bip_R_LowerArmをそれぞれアタッチする。

  3. VRMAvatarHandには　Handのオブジェクトの Armature.RightHandかArmature.LeftHandをそれぞれアタッチする。

HandRightとHandLeftのオブジェクトを OVRCameraRigのLeftContollerAnchorとRightControllerAnchorの下に設置する。

完成！

デモ

2022/05/08 追記

このままだと、Animation Clip時に正常に動かないので改良！

HandController.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class HandController : MonoBehaviour
{

    [SerializeField]
    private HandType.Hand HandType;

    private Transform[] AgentBones;
    private Transform[] AvatarBones;

	private Quaternion[] preAgentlocalRotations = new Quaternion[(int)VRMAgentHand.VRMBoneId.Hand_End]; 
    private Quaternion[] preAvatarlocalRotations = new Quaternion[(int)VRMAvatarHand.VRMBoneId.Hand_End];


    // Start is called before the first frame update
    public void Start()
    {
        var AgentHand = gameObject.GetComponent<VRMAgentHand>();
        AgentBones = AgentHand.GetHand(HandType);

        var AvatarHand = gameObject.GetComponent<VRMAvatarHand>();
        AvatarBones = AvatarHand.GetHand(HandType);

        if (AgentBones.Length != AvatarBones.Length){
            Debug.Log("Length is not equal with Agent hands and Avatar hands ->" + AgentBones.Length + " vs " + AvatarBones.Length);
        }
        for(int id = 1; id  < AgentBones.Length; id++){
            preAvatarlocalRotations[id] = AvatarBones[id].localRotation;
            preAgentlocalRotations[id] = AgentBones[id].localRotation;
        }        
    }

    // Update is called once per frame
    public void LateUpdate() 
    {
     for(int id = 1; id  < AvatarBones.Length; id++){
        var timeDiffAvatarRotation = preAvatarlocalRotations[id] * Quaternion.Inverse(AvatarBones[id].localRotation);
        AgentBones[id].localRotation = preAgentlocalRotations[id] * Quaternion.Inverse(timeDiffAvatarRotation);
        preAgentlocalRotations[id] = AgentBones[id].localRotation;
        preAvatarlocalRotations[id] = AvatarBones[id].localRotation;
        }   
    }
}

LateUpdate()でAnimation Clipによる更新後でも動くようになる。ただし、AgentのBoneのlocalRotationはAnimcationClipに保存されているものが使われちゃうので、preAgentLocalRotaionで保存しておく。

1
2
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
1
2

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?