UE4でWinRT APIを使ってウィンドウをゲーム内に表示する

UE4.26.2でWinRT APIを使ってキャプチャしたウィンドウをゲーム内で表示させてみたのでやったことをまとめておきます。

参照サイト

• Unreal での WinRT - Mixed Reality | Microsoft Docs
• HoloLens開発向けだがUnrealでWinRTを利用する手順について参照
• Windows 10のウィンドウキャプチャAPI - Qiita
• C++ から Windows Graphics Capture API を利用する方法について調べてみた - 凹みTips
• WinRTでのウィンドウキャプチャの実装について参照
• graphics-driver-samples/util.cpp at master · microsoft/graphics-driver-samples · GitHub
• ID3D11Texture2DをBMPに保存するサンプル、UTexture2Dにする部分で参照
• Use existing ID3D11Texture2D or OpenGL texture directly in Engine - Development Discussion / C++ Gameplay Programming - Unreal Engine Forums
• ID3D11Texture2Dを直接Utexture2Dとして利用できないかで参照(現状はできなさそう？)
• ayumax/WindowCapture2D: Library for capturing and displaying windows in real time with UnrealEngin
• GDIで実装されたウィンドウキャプチャプラグイン、UTexture2D更新まわりで参照

やったこと

C++でのキャプチャ部分実装

WinRT APIを使うためにBuild.csに以下の内容を追加しました。

[プロジェクト名].Build.cs

if (Target.Platform == UnrealTargetPlatform.Win64)
{
	bEnableExceptions = true;
	bUseUnity = false;
	CppStandard = CppStandardVersion.Cpp17;
	PublicSystemLibraries.AddRange(new string[] { "shlwapi.lib", "runtimeobject.lib", "D3D11.lib" });
	PrivateIncludePaths.Add(System.IO.Path.Combine(Target.WindowsPlatform.WindowsSdkDir,
												"Include",
												Target.WindowsPlatform.WindowsSdkVersion,
												"cppwinrt"));
}

キャプチャのコアクラスを実装、中身の詳細は上記の参照サイトを参照して下さい。
m_WinrtSizeに設定される値とUTexture2Dで作成時に指定する値が食い違っていたのでそれに合わせて実装しています。

WindowCapturer.h

#pragma once

#include "CoreMinimal.h"
#include "UObject/NoExportTypes.h"
#include "Engine/Texture2D.h"
#if PLATFORM_WINDOWS
#include "Windows/AllowWindowsPlatformTypes.h"
#include "Windows/AllowWindowsPlatformAtomics.h"
#include "Windows/PreWindowsApi.h"

#include <unknwn.h>
#include <winrt/base.h>
#include <winrt/Windows.Foundation.h>
#include <winrt/Windows.Foundation.Collections.h>
#include <winrt/Windows.Perception.Spatial.h>
#include <winrt/Windows.System.h>
#include <winrt/Windows.Graphics.h>
#include <winrt/Windows.Graphics.Capture.h>
#include <winrt/Windows.Graphics.DirectX.h>
#include <winrt/Windows.Graphics.DirectX.Direct3D11.h>
#include <d3d11.h>
#include <windows.graphics.capture.interop.h>
#include <windows.graphics.directx.direct3d11.interop.h>

#include "Windows/PostWindowsApi.h"
#include "Windows/HideWindowsPlatformAtomics.h"
#include "Windows/HideWindowsPlatformTypes.h"
#endif
#include "WindowCapturer.generated.h"

DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FWindowCapturerChangeTexture, UTexture2D*, NewTexture);

/**
 * 
 */
UCLASS(BlueprintType, Blueprintable)
class CAPWINDOWNEWAPI_API UWindowCapturer : public UObject
{
	GENERATED_BODY()
	
public:
	UWindowCapturer();
	virtual ~UWindowCapturer();

	UFUNCTION(BlueprintCallable)
	virtual void Start();

	UFUNCTION(BlueprintCallable)
	virtual void Close();

protected:
	void ReCreateTexture();

#if PLATFORM_WINDOWS
	winrt::Windows::Graphics::Capture::GraphicsCaptureItem CreateCaptureItem(HWND hwnd);
	winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice CreateDevice();
	void OnFrameArrived(winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, winrt::Windows::Foundation::IInspectable const& args);
	void UpdateTextureFromID3D11Texture2D(winrt::com_ptr<ID3D11Texture2D> texture);
#endif

public:
	UPROPERTY(BlueprintAssignable, Category = SceneCapture)
	FWindowCapturerChangeTexture ChangeTexture;

private:
	static bool IsInit;

	class UTexture2D* TextureTarget;

#if PLATFORM_WINDOWS
	HWND m_TargetWindow = nullptr;

	winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice m_WinrtDevice = nullptr;
	winrt::Windows::Graphics::Capture::GraphicsCaptureItem m_WinrtItem = nullptr;
	winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool m_WinrtFramePool = nullptr;
	winrt::Windows::Graphics::Capture::GraphicsCaptureSession m_WinrtSession = nullptr;
	winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::FrameArrived_revoker m_WinrtRevoker;
	winrt::Windows::Graphics::SizeInt32 m_WinrtSize = { 0, 0 };
	winrt::com_ptr<ID3D11Texture2D> m_WinrtTexture = nullptr;
	UINT m_Width = 0;
	UINT m_Height = 0;
#endif
};

WindowCapture.cpp

#include "WindowCapturer.h"

bool UWindowCapturer::IsInit = false;

UWindowCapturer::UWindowCapturer()
{
	if (!IsInit)
	{
		winrt::init_apartment(winrt::apartment_type::single_threaded);
		IsInit = true;
	}
}

UWindowCapturer::~UWindowCapturer()
{
	Close();
}

void UWindowCapturer::Start()
{
#if PLATFORM_WINDOWS
	try
	{
		if (!m_TargetWindow)
		{
			// アクティブウィンドウを決め打ちで取得
			m_TargetWindow = GetActiveWindow();
		}

		if (!m_TargetWindow)
		{
			return;
		}

		m_WinrtItem = CreateCaptureItem(m_TargetWindow);
		if (!m_WinrtItem)
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to CreateCaptureItem()"));
			return;
		}
		m_WinrtDevice = CreateDevice();
		if (!m_WinrtDevice)
		{
			UE_LOG(LogTemp, Error, TEXT("Failed to CreateDevice()"));
			return;
		}

		// StartCapture
		m_WinrtFramePool = winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::Create(
			m_WinrtDevice,
			winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
			2,
			m_WinrtItem.Size());

		m_WinrtRevoker = m_WinrtFramePool.FrameArrived(winrt::auto_revoke, { this, &UWindowCapturer::OnFrameArrived });
		m_WinrtSession = m_WinrtFramePool.CreateCaptureSession(m_WinrtItem);
		m_WinrtSession.StartCapture();
	}
	catch (const winrt::hresult_error& e)
	{
		const int code = e.code();
		const winrt::hstring message = e.message();
		UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
	}
#endif
}

void UWindowCapturer::Close()
{
#if PLATFORM_WINDOWS
	if (TextureTarget)
	{
		TextureTarget->ReleaseResource();
		TextureTarget = nullptr;
	}

	if (m_WinrtFramePool)
	{
		m_WinrtRevoker.revoke();
	}

	if (m_WinrtSession)
	{
		m_WinrtSession.Close();
		m_WinrtSession = nullptr;
	}

	if (m_WinrtFramePool)
	{
		m_WinrtFramePool.Close();
		m_WinrtFramePool = nullptr;
	}

	m_WinrtDevice = nullptr;
	m_WinrtItem = nullptr;
#endif
}

#if PLATFORM_WINDOWS
winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice UWindowCapturer::CreateDevice()
{
	try
	{
		UINT createDeviceFlags = D3D11_CREATE_DEVICE_FLAG::D3D11_CREATE_DEVICE_BGRA_SUPPORT;
		winrt::com_ptr<ID3D11Device> d3dDevice;
		winrt::check_hresult(D3D11CreateDevice(
			nullptr,
			D3D_DRIVER_TYPE_HARDWARE,
			nullptr,
			createDeviceFlags,
			nullptr,
			0,
			D3D11_SDK_VERSION,
			d3dDevice.put(),
			nullptr,
			nullptr));

		winrt::com_ptr<IDXGIDevice> dxgiDevice = d3dDevice.as<IDXGIDevice>();
		winrt::com_ptr<::IInspectable> device;
		winrt::check_hresult(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.get(), device.put()));

		return device.as<winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice>();
	}
	catch (const winrt::hresult_error& e)
	{
		const int code = e.code();
		const winrt::hstring message = e.message();
		UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
		return nullptr;
	}
}

winrt::Windows::Graphics::Capture::GraphicsCaptureItem UWindowCapturer::CreateCaptureItem(HWND hwnd)
{
	try
	{
		winrt::Windows::Foundation::IActivationFactory factory = winrt::get_activation_factory<winrt::Windows::Graphics::Capture::GraphicsCaptureItem>();
		winrt::impl::com_ref<::IGraphicsCaptureItemInterop> interop = factory.as<::IGraphicsCaptureItemInterop>();
		winrt::Windows::Graphics::Capture::GraphicsCaptureItem item{ nullptr };
		winrt::check_hresult(interop->CreateForWindow(m_TargetWindow, winrt::guid_of<ABI::Windows::Graphics::Capture::IGraphicsCaptureItem>(), reinterpret_cast<void**>(winrt::put_abi(item))));
		return item;
	}
	catch (const winrt::hresult_error& e)
	{
		const int code = e.code();
		const winrt::hstring message = e.message();
		UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
		return nullptr;
	}

}

void UWindowCapturer::OnFrameArrived(winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, winrt::Windows::Foundation::IInspectable const& args)
{
	try
	{
		winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame frame = sender.TryGetNextFrame();
		if (!frame)
		{
			UE_LOG(LogTemp, Warning, TEXT("Failed to TryGetNextFrame() on OnFrameArrived"));
			return;
		}

		winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DSurface surface = frame.Surface();
		if (!surface)
		{
			UE_LOG(LogTemp, Warning, TEXT("Failed to Surface() on OnFrameArrived"));
			return;
		}

		winrt::impl::com_ref<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> access =
			surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>();
		winrt::check_hresult(access->GetInterface(winrt::guid_of<ID3D11Texture2D>(), m_WinrtTexture.put_void()));

		winrt::Windows::Graphics::SizeInt32 Size = frame.ContentSize();
		if ((m_WinrtSize.Height != Size.Height) || (m_WinrtSize.Width != Size.Width))
		{
			m_WinrtSize = Size;
			m_WinrtFramePool.Recreate(
				m_WinrtDevice,
				winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
				2,
				m_WinrtSize);
		}

		UpdateTextureFromID3D11Texture2D(m_WinrtTexture);
		ChangeTexture.Broadcast(TextureTarget);
	}
	catch (const winrt::hresult_error& e)
	{
		const int code = e.code();
		const winrt::hstring message = e.message();
		UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
	}
}

void UWindowCapturer::UpdateTextureFromID3D11Texture2D(winrt::com_ptr<ID3D11Texture2D> Texture)
{
	try
	{
		// First verify that we can map the texture
		D3D11_TEXTURE2D_DESC desc;
		Texture->GetDesc(&desc);

		// Get the device context
		winrt::com_ptr<ID3D11Device> d3dDevice;
		Texture->GetDevice(d3dDevice.put());
		winrt::com_ptr<ID3D11DeviceContext> d3dContext;
		d3dDevice->GetImmediateContext(d3dContext.put());

		// map the texture
		winrt::com_ptr<ID3D11Texture2D> mappedTexture;
		D3D11_MAPPED_SUBRESOURCE mapInfo;
		mapInfo.RowPitch;
		HRESULT hr = d3dContext->Map(
			Texture.get(),
			0,  // Subresource
			D3D11_MAP_READ,
			0,  // MapFlags
			&mapInfo);
		if (FAILED(hr))
		{
			if (hr == E_INVALIDARG)
			{
				D3D11_TEXTURE2D_DESC desc2;
				desc2.Width = desc.Width;
				desc2.Height = desc.Height;
				desc2.MipLevels = desc.MipLevels;
				desc2.ArraySize = desc.ArraySize;
				desc2.Format = desc.Format;
				desc2.SampleDesc = desc.SampleDesc;
				desc2.Usage = D3D11_USAGE_STAGING;
				desc2.BindFlags = 0;
				desc2.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
				desc2.MiscFlags = 0;

				winrt::com_ptr<ID3D11Texture2D> stagingTexture;
				winrt::check_hresult(d3dDevice->CreateTexture2D(&desc2, nullptr, stagingTexture.put()));

				// copy the texture to a staging resource
				d3dContext->CopyResource(stagingTexture.get(), Texture.get());

				// now, map the staging resource
				winrt::check_hresult(d3dContext->Map(
					stagingTexture.get(),
					0,
					D3D11_MAP_READ,
					0,
					&mapInfo));

				mappedTexture = std::move(stagingTexture);
			}
			else
			{
				winrt::check_hresult(hr);
			}
		}
		else
		{
			mappedTexture = Texture;
		}

		if (!m_WinrtTexture) return;

		UINT Width = desc.Width;
		UINT Height = desc.Height;

		if ((m_Width != Width) ||
			(m_Height != Height))
		{
			m_Width = Width;
			m_Height = Height;
			ReCreateTexture();
		}

		auto Region = new FUpdateTextureRegion2D(0, 0, 0, 0, m_Width, m_Height);
		TextureTarget->UpdateTextureRegions(0, 1, Region, mapInfo.RowPitch, 4, (uint8*)mapInfo.pData);
	}
	catch (const winrt::hresult_error& e)
	{
		const int code = e.code();
		const winrt::hstring message = e.message();
		UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
	}
}
#endif

void UWindowCapturer::ReCreateTexture()
{
#if PLATFORM_WINDOWS
	if (m_Height == 0 || m_Width == 0)
	{
		TextureTarget = nullptr;
		return;
	}

	TextureTarget = UTexture2D::CreateTransient(m_Width, m_Height, PF_B8G8R8A8);
	TextureTarget->UpdateResource();
#endif
}

BlueprintでのC++クラスの利用

C++で実装したクラスを使ってStaticMeshでPlaneを持つアクタでテクスチャを表示しています。

WindowCapture2DプラグインのWinRT API版

WindowCapture2DプラグインをForkしてWinRT APIを使ってウィンドウキャプチャするように改造してみました。
(GDI版は古いWindowsもサポートしている&今のやり方だとUE4.26以降しかサポートしていないと思うのでpullreqはしていません、切り替えられたらいいのかな？)
https://github.com/mechamogera/WindowCapture2D

対象ウィンドウが閉じられたことを検知する

以下のようにwinrt::Windows::Graphics::Capture::GraphicsCaptureItem::Closedのイベントによって対象ウィンドウが閉じられたことを検知できるみたいです。

WindowCapture.cpp

{
	...
	m_WinrtItem.Closed({ this, &UWindowCapturer::OnTargetClosed });
	...
}
...
void UWindowCapturer::OnTargetClosed(winrt::Windows::Graphics::Capture::GraphicsCaptureItem Item, winrt::Windows::Foundation::IInspectable Inspectable)
{

フレームレートを指定してキャプチャする

FrameArrivedを利用せず以下のようにすることで指定したフレームレートでキャプチャできるみたいです。

WindowCapture.h

#include "Containers/Ticker.h"
...
UCLASS(BlueprintType, Blueprintable)
class CAPWINDOWNEWAPI_API UWindowCapturer : public UObject
{
...
	bool Tick(float deltaTime);
...
	FTickerDelegate TickDelegate;
	FDelegateHandle TickHandle;
...
}

WindowCapture.cpp

void UWindowCapturer::Start()
{
...
		if (TickHandle.IsValid())
		{
			FTicker::GetCoreTicker().RemoveTicker(TickHandle);
			TickHandle.Reset();
		}
		TickDelegate = FTickerDelegate::CreateUObject(this, &UWindowCapturer::Tick);
		TickHandle = FTicker::GetCoreTicker().AddTicker(TickDelegate, 1.0f / 10.0f;
...
}

void UWindowCapturer::Close()
{
#if PLATFORM_WINDOWS
	if (TickHandle.IsValid())
	{
		FTicker::GetCoreTicker().RemoveTicker(TickHandle);
		TickHandle.Reset();
	}
...
}
...
bool UWindowCapturer::Tick(float deltaTime)
{
	OnFrameArrived(m_WinrtFramePool, nullptr);

	return true;
}