ネイティブDLLとか出回ってるラッパーライブラリとかに頼らず、C#だけでかつ自力で何とかサクッとXAudio2を扱えないものかと思ってたが、頑張ったら出来たのでご紹介。

頑張ったところ

大きなポイントは下記。

IXAudio2はCOMインターフェースなので、比較的扱いやすかった。ただXAudio2Createで得られるのがポインタなので、GetTypedObjectForIUnknownで実体に変換して使った。
一方、IXAudio2以外のインターフェースがほぼCOMではないので、ポインタから関数テーブルをほじくり出して、使うメソッドだけGetDelegateForFunctionPointerでデリゲートに変換して無理矢理動かした。ちなみにこのやり方だとクラス（インターフェース）のメソッドは、引数の先頭にクラスのインスタンスのポインタを追加する必要がある。静的メソッドだと無くなるのかな。
IXAudio2VoiceCallbackはCreateSourceVoiceに継承クラスを丸ごと渡す仕様なので、クラスどうやってネイティブに渡すんだ…詰んだ…と思ったけど、さっきとは逆の発想で、実装側のコールバック関数を職人が1個1個丁寧にデリゲートにしてGetFunctionPointerForDelegateでポインタ化、関数テーブルにまとめてまたポインタ作ってCreateSourceVoiceに恐る恐る渡してみたら何と動いたので勝てば官軍。

サンプルコード

XAudio2.cs … ややこしい処理をラップしてまとめたやつ
Program.cs … 1秒間音を鳴らすサンプル。コンソールアプリ

※XAudio2.9を使用しているので、Windows 10専用。
※今回使わなかったメソッドはちゃんと定義してない。
※「XAudio2.cs」の中、コメントで大きく囲ってある部分は、Voice系Interfaceの本来の定義の概略。関数テーブルの順番と併記してる数字が対応してる（入れ替わると動かない）ので、今回のサンプルで使ってないメソッドを使いたい場合の参考にしていただければ。
※パッションの赴くまま組んだので、例外処理等一切無かったり、アンマネージ対応が中途半端だったりする。流石にこれはヤバいよってのがあればこっそり教えてください。
※手探りで組んだので、もっと良い方法があったり、そもそも正しいコードかどうかも分からないのでご了承を。

XAudio2.cs

using System;
using System.Runtime.InteropServices;	//Guid, InterfaceType, Marshal, DllImport
using System.Runtime.Versioning;    //SupportedOSPlatform

[assembly:SupportedOSPlatform("windows")]
namespace XAudio2CS {
    [StructLayout(LayoutKind.Sequential, Pack = 4)]
    internal struct WAVEFORMATEX {
        internal ushort wFormatTag;
        internal ushort nChannels;
        internal uint nSamplesPerSec;
        internal uint nAvgBytesPerSec;
        internal ushort nBlockAlign;
        internal ushort wBitsPerSample;
        internal ushort cbSize;
    }

    [StructLayout(LayoutKind.Sequential, Pack = 4)]
	internal struct XAUDIO2_BUFFER {
		internal uint Flags;
		internal uint AudioBytes;
		internal IntPtr pAudioData;
		internal uint PlayBegin;
		internal uint PlayLength;
		internal uint LoopBegin;
		internal uint LoopLength;
		internal uint LoopCount;
		internal IntPtr pContext;
	}

	[Guid("2B02E3CF-2E0B-4ec3-BE45-1B2A3FE7210D"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
	internal interface IXAudio2 {
		int RegisterForCallbacks();	//dummy
		void UnregisterForCallbacks();	//dummy
		int CreateSourceVoice(out IntPtr ppSourceVoice, WAVEFORMATEX wfex, uint Flags, float MaxFrequencyRatio, IntPtr pCallback, IntPtr pSendList, IntPtr pEffectChain);
		int CreateSubmixVoice();	//dummy
		int CreateMasteringVoice(out IntPtr ppMasteringVoice, uint InputChannels, uint InputSampleRate, uint Flags, string szDeviceId, IntPtr pEffectChain, uint StreamCategory);
		int StartEngine();	//dummy
		void StopEngine();	//dummy
		int CommitChanges();	//dummy
		void GetPerformanceData();	//dummy
		void SetDebugConfiguration();	//dummy
	}
/*
	internal interface IXAudio2Voice {
		void GetVoiceDetails();	//dummy0
		int SetOutputVoices();	//dummy1
		int SetEffectChain();	//dummy2
		int EnableEffect();	//dummy3
		int DisableEffect();	//dummy4
		void GetEffectState();	//dummy5
		int SetEffectParameters();	//dummy6
		int GetEffectParameters();	//dummy7
		int SetFilterParameters();	//dummy8
		void GetFilterParameters();	//dummy9
		int SetOutputFilterParameters();	//dummy10
		void GetOutputFilterParameters();	//dummy11
		int SetVolume();	//dummy12
		void GetVolume();	//dummy13
		int SetChannelVolumes();	//dummy14
		void GetChannelVolumes();	//dummy15
		int SetOutputMatrix();	//dummy16
		void GetOutputMatrix();	//dummy17
		void DestroyVoice();	//dummy18
	}

	internal interface IXAudio2SourceVoice : IXAudio2Voice {
		int Start();	//dummy19
		int Stop();	//dummy20
		int SubmitSourceBuffer();	//dummy21
		int FlushSourceBuffers();	//dummy22
		int Discontinuity();	//dummy23
		int ExitLoop();	//dummy24
		void GetState();	//dummy25
		int SetFrequencyRatio();	//dummy26
		void GetFrequencyRatio();	//dummy27
		int SetSourceSampleRate();	//dummy28
	}

	internal interface IXAudio2MasteringVoice : IXAudio2Voice {
		int GetChannelMask();	//dummy19
	}
*/
	internal class XAudio2 {
		internal const uint XAUDIO2_DEFAULT_PROCESSOR = 0x1;
        internal const ushort WAVE_FORMAT_PCM = 1;
		private static IXAudio2 xaudio2 = null;

		internal static int Create(uint Flags = 0, uint XAudio2Processor = XAUDIO2_DEFAULT_PROCESSOR) {
			if (xaudio2 != null) return -1;
			var hr = NativeMethods.XAudio2Create(out var pXAudio2, Flags, XAudio2Processor);
			xaudio2 = (IXAudio2)Marshal.GetTypedObjectForIUnknown(pXAudio2, typeof(IXAudio2));
			return hr;
		}

		internal static int Release() {
			if (xaudio2 == null) return -1;
            var pobj = Marshal.GetIUnknownForObject(xaudio2);
            var ret = Marshal.Release(pobj);
			xaudio2 = null;
			return ret;
		}

		internal static int CreateMasteringVoice(out XAudio2MasteringVoice masteringvoice, uint InputChannels = 0, uint InputSampleRate = 0, uint Flags = 0, string szDeviceId = null, IntPtr pEffectChain = default(IntPtr), uint StreamCategory = 6) {
			var hr = xaudio2.CreateMasteringVoice(out var pmasteringvoice, InputChannels, InputSampleRate, Flags, szDeviceId, pEffectChain, StreamCategory);
			masteringvoice = new XAudio2MasteringVoice(pmasteringvoice);
			return hr;
		}

		internal static int CreateSourceVoice(out XAudio2SourceVoice sourcevoice, WAVEFORMATEX wfex, uint Flags = 0, float MaxFrequencyRatio = 2.0f, IntPtr pCallback = default(IntPtr), IntPtr pSendList = default(IntPtr), IntPtr pEffectChain = default(IntPtr)) {
			var hr = xaudio2.CreateSourceVoice(out var psourcevoice, wfex, Flags, MaxFrequencyRatio, pCallback, pSendList, pEffectChain);
			sourcevoice = new XAudio2SourceVoice(psourcevoice);
			return hr;
		}
	}

	internal class XAudio2MasteringVoice {
		private NativeMethods.DestroyVoice _DestroyVoice;
		private IntPtr m_pmasteringvoice;
		private IntPtr[] pfuncs = new IntPtr[20];

		internal XAudio2MasteringVoice(IntPtr pmasteringvoice) {
			m_pmasteringvoice = pmasteringvoice;
			Marshal.Copy(Marshal.ReadIntPtr(pmasteringvoice, 0), pfuncs, 0, pfuncs.Length);
			_DestroyVoice = Marshal.GetDelegateForFunctionPointer<NativeMethods.DestroyVoice>(pfuncs[18]);
		}

		internal void DestroyVoice() {
			_DestroyVoice(m_pmasteringvoice);
		}
	}

	internal class XAudio2SourceVoice {
		private NativeMethods.SetVolume _SetVolume;
		private NativeMethods.DestroyVoice _DestroyVoice;
		private NativeMethods.Start _Start;
		private NativeMethods.Stop _Stop;
		private NativeMethods.SubmitSourceBuffer _SubmitSourceBuffer;
		private NativeMethods.FlushSourceBuffers _FlushSourceBuffers;
		private IntPtr m_psourcevoice;
		private IntPtr[] pfuncs = new IntPtr[29];

		internal XAudio2SourceVoice(IntPtr psourcevoice) {
			m_psourcevoice = psourcevoice;
			Marshal.Copy(Marshal.ReadIntPtr(psourcevoice, 0), pfuncs, 0, pfuncs.Length);

			_SetVolume = Marshal.GetDelegateForFunctionPointer<NativeMethods.SetVolume>(pfuncs[12]);
			_DestroyVoice = Marshal.GetDelegateForFunctionPointer<NativeMethods.DestroyVoice>(pfuncs[18]);
			_Start = Marshal.GetDelegateForFunctionPointer<NativeMethods.Start>(pfuncs[19]);
			_Stop = Marshal.GetDelegateForFunctionPointer<NativeMethods.Stop>(pfuncs[20]);
			_SubmitSourceBuffer = Marshal.GetDelegateForFunctionPointer<NativeMethods.SubmitSourceBuffer>(pfuncs[21]);
			_FlushSourceBuffers = Marshal.GetDelegateForFunctionPointer<NativeMethods.FlushSourceBuffers>(pfuncs[22]);
		}

		internal int SetVolume(float Volume, uint OperationSet = 0) {
			return (_SetVolume(m_psourcevoice, Volume, OperationSet));
		}

		internal void DestroyVoice() {
			_DestroyVoice(m_psourcevoice);
		}

		internal int Start(uint Flags = 0, uint OperationSet = 0) {
			return (_Start(m_psourcevoice, Flags, OperationSet));
		}

		internal int Stop(uint Flags = 0, uint OperationSet = 0) {
			return (_Stop(m_psourcevoice, Flags, OperationSet));
		}

		internal int SubmitSourceBuffer(IntPtr pBuffer, IntPtr pBufferWMA = default(IntPtr)) {
			return (_SubmitSourceBuffer(m_psourcevoice, pBuffer, pBufferWMA));
		}

		internal int FlushSourceBuffers() {
			return (_FlushSourceBuffers(m_psourcevoice));
		}
	}

	internal abstract class XAudio2VoiceCallback {
        internal IntPtr funcentry { get; }

		private delegate void dlg_OnVoiceProcessingPassStart(IntPtr self, uint BytesRequired);
		private delegate void dlg_OnVoiceProcessingPassEnd(IntPtr self);
		private delegate void dlg_OnStreamEnd(IntPtr self);
		private delegate void dlg_OnBufferStart(IntPtr self, IntPtr pBufferContext);
		private delegate void dlg_OnBufferEnd(IntPtr self, IntPtr pBufferContext);
		private delegate void dlg_OnLoopEnd(IntPtr self, IntPtr pBufferContext);
		private delegate void dlg_OnVoiceError(IntPtr self, IntPtr pBufferContext, int Error);
        private IntPtr[] callbackfuncs = new IntPtr[7];
        private IntPtr functable;

        internal XAudio2VoiceCallback() {
            callbackfuncs[0] = Marshal.GetFunctionPointerForDelegate<dlg_OnVoiceProcessingPassStart>(_OnVoiceProcessingPassStart);
            callbackfuncs[1] = Marshal.GetFunctionPointerForDelegate<dlg_OnVoiceProcessingPassEnd>(_OnVoiceProcessingPassEnd);
            callbackfuncs[2] = Marshal.GetFunctionPointerForDelegate<dlg_OnStreamEnd>(_OnStreamEnd);
            callbackfuncs[3] = Marshal.GetFunctionPointerForDelegate<dlg_OnBufferStart>(_OnBufferStart);
            callbackfuncs[4] = Marshal.GetFunctionPointerForDelegate<dlg_OnBufferEnd>(_OnBufferEnd);
            callbackfuncs[5] = Marshal.GetFunctionPointerForDelegate<dlg_OnLoopEnd>(_OnLoopEnd);
            callbackfuncs[6] = Marshal.GetFunctionPointerForDelegate<dlg_OnVoiceError>(_OnVoiceError);
            functable = Marshal.AllocCoTaskMem(Marshal.SizeOf<IntPtr>() * callbackfuncs.Length);
            Marshal.Copy(callbackfuncs, 0, functable, callbackfuncs.Length);
            funcentry = Marshal.AllocCoTaskMem(Marshal.SizeOf<IntPtr>());
            Marshal.WriteIntPtr(funcentry, 0, functable);
        }

        internal void Release() {
            Marshal.FreeCoTaskMem(funcentry);
            Marshal.FreeCoTaskMem(functable);
        }

        private void _OnVoiceProcessingPassStart(IntPtr self, uint BytesRequired) {
			OnVoiceProcessingPassStart(BytesRequired);
		}

        private void _OnVoiceProcessingPassEnd(IntPtr self) {
			OnVoiceProcessingPassEnd();
		}

		private void _OnStreamEnd(IntPtr self) {
			OnStreamEnd();
		}

		private void _OnBufferStart(IntPtr self, IntPtr pBufferContext) {
			OnBufferStart(pBufferContext);
		}

		private void _OnBufferEnd(IntPtr self, IntPtr pBufferContext) {
			OnBufferEnd(pBufferContext);
		}
		
		private void _OnLoopEnd(IntPtr self, IntPtr pBufferContext) {
			OnLoopEnd(pBufferContext);
		}

		private void _OnVoiceError(IntPtr self, IntPtr pBufferContext, int Error) {
			OnVoiceError(pBufferContext, Error);
		}

        internal abstract void OnVoiceProcessingPassStart(uint BytesRequired);
        internal abstract void OnVoiceProcessingPassEnd();
		internal abstract void OnStreamEnd();
		internal abstract void OnBufferStart(IntPtr pBufferContext);
		internal abstract void OnBufferEnd(IntPtr pBufferContext);
		internal abstract void OnLoopEnd(IntPtr pBufferContext);
		internal abstract void OnVoiceError(IntPtr pBufferContext, int Error);
	}

	internal static class NativeMethods {
		[DllImport("xaudio2_9.dll")]
		internal static extern int XAudio2Create(out IntPtr ppXAudio2, uint Flags, uint XAudio2Processor);

		[UnmanagedFunctionPointer(CallingConvention.ThisCall)]
		internal delegate int SetVolume(IntPtr self, float Volume, uint OperationSet);

		[UnmanagedFunctionPointer(CallingConvention.ThisCall)]
		internal delegate void DestroyVoice(IntPtr self);

		[UnmanagedFunctionPointer(CallingConvention.ThisCall)]
		internal delegate int Start(IntPtr self, uint Flags, uint OperationSet);

		[UnmanagedFunctionPointer(CallingConvention.ThisCall)]
		internal delegate int Stop(IntPtr self, uint Flags, uint OperationSet);

		[UnmanagedFunctionPointer(CallingConvention.ThisCall)]
		internal delegate int SubmitSourceBuffer(IntPtr self, IntPtr pBuffer, IntPtr pBufferWMA);

		[UnmanagedFunctionPointer(CallingConvention.ThisCall)]
		internal delegate int FlushSourceBuffers(IntPtr self);
	}
}

Program.cs

using System;
using System.Runtime.InteropServices; //GCHandle
using System.Threading; //EventWaitHandle
using XAudio2CS;

namespace xaudio2test
{
    class Program
    {
        private static XAudio2MasteringVoice m_masteringvoice;
        private static XAudio2SourceVoice m_sourcevoice;
        private static byte[] wavedata;
        private static GCHandle gch_wavedata;
        private static XAUDIO2_BUFFER buf;
        private static GCHandle gch_buf;
        private static VoiceCallback m_callback;

        static void Main(string[] args)
        {
            XAudio2.Create();
            XAudio2.CreateMasteringVoice(out m_masteringvoice);

            var wfex = new WAVEFORMATEX();
            ushort channels = 2;
            uint samplerate = 48000;
            wfex.wFormatTag = XAudio2.WAVE_FORMAT_PCM;
            wfex.nChannels = channels;
            wfex.nSamplesPerSec = samplerate;
            wfex.nBlockAlign = (ushort)((16 / 8) * channels);
            wfex.nAvgBytesPerSec = samplerate * (uint)((16 / 8) * channels);
            wfex.wBitsPerSample = 16;
            m_callback = new VoiceCallback();
            XAudio2.CreateSourceVoice(out m_sourcevoice, wfex, 0, 2.0f, m_callback.funcentry);

            m_sourcevoice.SetVolume(0.2f);
            m_sourcevoice.Start();

            wavedata = new byte[48000 * 2 * 2];
            uint uintval;
            for (var i = 0; i < 48000; ++i) {
                uintval = (uint)(i % 110) * 595;
                wavedata[i * 4] = (byte)(uintval & 0xff);
                wavedata[i * 4 + 1] = (byte)(uintval >> 8);
                uintval = (uint)(i % 73) * 892;
                wavedata[i * 4 + 2] = (byte)(uintval & 0xff);
                wavedata[i * 4 + 3] = (byte)(uintval >> 8);
            }
            gch_wavedata = GCHandle.Alloc(wavedata, GCHandleType.Pinned);
            buf = new XAUDIO2_BUFFER();
            buf.pAudioData = gch_wavedata.AddrOfPinnedObject();
            buf.AudioBytes = (uint)(wavedata.Length);
            gch_buf = GCHandle.Alloc(buf, GCHandleType.Pinned);
            m_sourcevoice.SubmitSourceBuffer(gch_buf.AddrOfPinnedObject());

            m_callback.m_event_BufferEnd.WaitOne();
            Console.WriteLine("バッファ終了。");

            m_sourcevoice.Stop();
            m_sourcevoice.FlushSourceBuffers();
            gch_buf.Free();
            gch_wavedata.Free();
            m_sourcevoice.DestroyVoice();
            m_callback.Release();
            m_masteringvoice.DestroyVoice();
            XAudio2.Release();
        }
    }

    class VoiceCallback : XAudio2VoiceCallback {
        internal EventWaitHandle m_event_BufferEnd { get; }

        internal VoiceCallback() {
            m_event_BufferEnd = new EventWaitHandle(false, EventResetMode.AutoReset);
        }

        internal override void OnVoiceProcessingPassStart(uint BytesRequired) {}
        internal override void OnVoiceProcessingPassEnd() {}
		internal override void OnStreamEnd() {}
		internal override void OnBufferStart(IntPtr pBufferContext) {}

		internal override void OnBufferEnd(IntPtr pBufferContext) {
            m_event_BufferEnd.Set();
        }

		internal override void OnLoopEnd(IntPtr pBufferContext) {}
		internal override void OnVoiceError(IntPtr pBufferContext, int Error) {}
    }
}

追記

実行した途端に左と右から高さの違う音（ノコギリ波）が鳴り始めます。1秒後鳴り止みます。
いきなり大きな音が出るとビックリすると思ったので、SetVolumeで音量を0.2に絞ってあります。お好みで調整してください。（0:無音～ 1:最大）
WinAPIのWaveOut系関数だとC#でもDllImportするだけで簡単に音鳴らせるけど、それでもXAudio2を使いたい理由としては、複数サウンドを重ねて再生出来る（試してないけど）のと、音量調整がWindowsの音量ミキサーとは独立してること。（waveOutSetVolumeで音量いじると音量ミキサーのつまみが動いちゃう。）
VSCode上でデバッグで実行すると左スピーカーからしか音が出ないのはオレ環？XAudio2もだし、WaveOut系いじってた時もそうだった。