HoloLensアドベントカレンダー2021の6日目の記事です。
MicStreamSelectorを使いたいという要望があったので、MRTKのサンプルにあるWindowsMicrophoneStreamDemoをやってみました。

開発環境

HoloLens2
Unity 2020.3.0f1
MRTK 2.7.2
MicStreamSelector commit d7cb7dcd4ce18437cc81789e6e364f4beb56dea8

実装

1．MRTKをunitypackageから入れます

Microsoft.MixedReality.Toolkit.Unity.Examples.2.7.2.unitypackage
Microsoft.MixedReality.Toolkit.Unity.Extensions.2.7.2.unitypackage
Microsoft.MixedReality.Toolkit.Unity.Foundation.2.7.2.unitypackage
Microsoft.MixedReality.Toolkit.Unity.TestUtilities.2.7.2.unitypackage
Microsoft.MixedReality.Toolkit.Unity.Tools.2.7.2.unitypackage

2．シーンのWindowsMicrophoneStreamDemoを開きます

2．MicStreamSelectorをVS2019、Release、ARM64でビルドします

3．できたDLLとMicStreamSelector->UnityAddonにある3つのスクリプトをUnityプロジェクトに追加します

4．SceneContentのInspectorビューにあるMicrophoneAmplitudeDemo.csを編集します

#if MICSTREAM_PRESENT
#end
をコメントアウトします。

wireThicknessの値をDebugTextに表示しています

MicrophoneAmplitudeDemo.cs

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using Microsoft.MixedReality.Toolkit.Audio;
using Microsoft.MixedReality.Toolkit.SpatialAwareness;
using UnityEngine;
using TMPro;

namespace Microsoft.MixedReality.Toolkit.Examples
{
    /// <summary>
    /// Demonstration class using WindowsMicrophoneStream (from com.microsoft.mixedreality.toolkit.micstream) to select the 
    /// voice microphone and adjust the spatial awareness mesh based on the amplitude of the user's voice.
    /// </summary>
    [RequireComponent(typeof(AudioSource))]
    public class MicrophoneAmplitudeDemo : MonoBehaviour
    {
// #if MICSTREAM_PRESENT
        public TextMeshProUGUI debugText;

        [SerializeField]
        [Tooltip("Gain to apply to the microphone input.")]
        [Range(0, 10)]
        private float inputGain = 1.0f;

        [SerializeField]
        [Tooltip("Factor by which to boost the microphone amplitude when changing the mesh display.")]
        [Range(0, 50)]
        private int amplitudeBoostFactor = 10;

        [SerializeField]
        [Tooltip("Color to use for the wireframe mesh.\nIt is recommended to use a color with an alpha of 255.")]
        private Color meshColor = Color.blue;

        private IMixedRealitySpatialAwarenessMeshObserver spatialMeshObserver = null;
        private Material visibleMaterial = null;

        /// <summary>
        /// Class providing microphone stream management support on Microsoft Windows based devices.
        /// </summary>
        private WindowsMicrophoneStream micStream = null;

        /// <summary>
        /// The average amplitude of the sound captured during the most recent microphone update.
        /// </summary>
        private float averageAmplitude = 0.0f;

        /// <summary>
        /// Cached material values used to restore initial settings when running the demo in the editor.
        /// </summary>
        private Color defaultMaterialColor = Color.black;
        private int defaultWireThickness = 0;

        private void Awake()
        {
            // We do not wish to play the ambient room sound from the audio source.
            gameObject.GetComponent<AudioSource>().volume = 0.0f;

            spatialMeshObserver = (CoreServices.SpatialAwarenessSystem as IMixedRealityDataProviderAccess)?.GetDataProvider<IMixedRealitySpatialAwarenessMeshObserver>();
            visibleMaterial = spatialMeshObserver?.VisibleMaterial;

            if (visibleMaterial != null)
            {
                // Cache the initial material settings.
                defaultMaterialColor = visibleMaterial.GetColor("_WireColor");
                defaultWireThickness = visibleMaterial.GetInt("_WireThickness");

                visibleMaterial.SetColor("_WireColor", meshColor);
            }

            micStream = new WindowsMicrophoneStream();
            if (micStream == null)
            {
                Debug.Log("Failed to create the Windows Microphone Stream object");
            }

            micStream.Gain = inputGain;

            // Initialize the microphone stream.
            WindowsMicrophoneStreamErrorCode result = micStream.Initialize(WindowsMicrophoneStreamType.HighQualityVoice);
            if (result != WindowsMicrophoneStreamErrorCode.Success)
            {
                Debug.Log($"Failed to initialize the microphone stream. {result}");
                return;
            }

            // Start the microphone stream.
            // Do not keep the data and do not preview.
            result = micStream.StartStream(false, false);
            if (result != WindowsMicrophoneStreamErrorCode.Success)
            {
                Debug.Log($"Failed to start the microphone stream. {result}");
            }
        }

        private void OnDestroy()
        {
            if (micStream == null) { return; }

            // Stop the microphone stream.
            WindowsMicrophoneStreamErrorCode result = micStream.StopStream();
            if (result != WindowsMicrophoneStreamErrorCode.Success)
            {
                Debug.Log($"Failed to stop the microphone stream. {result}");
            }

            // Uninitialize the microphone stream.
            micStream.Uninitialize();
            micStream = null;

            // Restore the initial material settings.
            if (visibleMaterial != null)
            {
                visibleMaterial.SetColor("_WireColor", defaultMaterialColor);
                visibleMaterial.SetInt("_WireThickness", defaultWireThickness);
            }
        }

        private void OnDisable()
        {
            if (micStream == null) { return; }

            // Pause the microphone stream.
            WindowsMicrophoneStreamErrorCode result = micStream.Pause();
            if (result != WindowsMicrophoneStreamErrorCode.Success)
            {
                Debug.Log($"Failed to pause the microphone stream. {result}");
            }
        }

        private void OnEnable()
        {
            if (micStream == null) { return; }

            // Resume the microphone stream.
            WindowsMicrophoneStreamErrorCode result = micStream.Resume();
            if (result != WindowsMicrophoneStreamErrorCode.Success)
            {
                Debug.Log($"Failed to resume the microphone stream. {result}");
            }
        }

        private static int maxWireThickness = 750;

        private void Update()
        {
            if (micStream == null) { return; }

            // Update the gain, if changed.
            if (micStream.Gain != inputGain)
            {
                micStream.Gain = inputGain;
            }

            if (visibleMaterial != null)
            {
                // Artificially increase the amplitude to make the visible effect more pronounced.
                int wireThickness = (int)(averageAmplitude * amplitudeBoostFactor * maxWireThickness);
                wireThickness = Mathf.Clamp(wireThickness, 0, maxWireThickness);
                visibleMaterial.SetInt("_WireThickness", wireThickness);
                debugText.text = wireThickness.ToString();
            }
        }

        private void OnAudioFilterRead(float[] buffer, int numChannels)
        {
            if (micStream == null) { return; }

            // Read the microphone stream data.
            WindowsMicrophoneStreamErrorCode result = micStream.ReadAudioFrame(buffer, numChannels);
            if (result != WindowsMicrophoneStreamErrorCode.Success)
            {
                Debug.Log($"Failed to read the microphone stream data. {result}");
            }

            float sumOfValues = 0;

            // Calculate this frame's average amplitude.
            for (int i = 0; i < buffer.Length; i++)
            {
                if (float.IsNaN(buffer[i]))
                {
                    buffer[i] = 0;
                }

                buffer[i] = Mathf.Clamp(buffer[i], -1.0f, 1.0f);
                sumOfValues += Mathf.Clamp01(Mathf.Abs(buffer[i]));
            }

            averageAmplitude = sumOfValues / buffer.Length;
        }

// #endif // MICSTREAM_PRESENT
    }
}

5．Main Cameraの下にTextMeshProのTextを作成して、SceneContentのDebugTextにアタッチしてください

6．MicStreamSelectorの設定

7．Build Settingsの設定

8．ProjectSettings->Player->Publishing Settings->Capabilitiesのチェック項目（デプロイ前にPackage.appxmanifestの機能に反映されているかもチェック）

InternetClient
WebCam
Microphone
SpatialPerception
GazeInput

9．XR Plug-in Managementの設定（※今回、OpenXRは使いません）

10．ビルドして、Release、ARM64でHoloLensにデプロイしましょう

デモ

ボイスの大きさに合わせて、空間メッシュのワイヤーフレームが太くなるみたいです！

＊音量注意
WindowsMicrophoneStreamDemo#HoloLens2 #MRTK pic.twitter.com/WEM12bUJKn
— がちもとさん@メタバース熊本 (@sotongshi) December 3, 2021

ただし、アプリ内で音出しちゃうとマイク機能が使えなくなるバグがあります。録画不可です。一度マイク機能が使えなくなると、アプリ再起動しても無理です。本体を再起動したらいけます。たぶん起動時にmicStreamのリセットが必要だと思います。

お疲れ様でした。