UE4.26.2でWinRT APIを使ってキャプチャしたウィンドウをゲーム内で表示させてみたのでやったことをまとめておきます。
参照サイト
-
Unreal での WinRT - Mixed Reality | Microsoft Docs
- HoloLens開発向けだがUnrealでWinRTを利用する手順について参照
- Windows 10のウィンドウキャプチャAPI - Qiita
-
C++ から Windows Graphics Capture API を利用する方法について調べてみた - 凹みTips
- WinRTでのウィンドウキャプチャの実装について参照
-
graphics-driver-samples/util.cpp at master · microsoft/graphics-driver-samples · GitHub
- ID3D11Texture2DをBMPに保存するサンプル、UTexture2Dにする部分で参照
-
Use existing ID3D11Texture2D or OpenGL texture directly in Engine - Development Discussion / C++ Gameplay Programming - Unreal Engine Forums
- ID3D11Texture2Dを直接Utexture2Dとして利用できないかで参照(現状はできなさそう？)
-
ayumax/WindowCapture2D: Library for capturing and displaying windows in real time with UnrealEngin
- GDIで実装されたウィンドウキャプチャプラグイン、UTexture2D更新まわりで参照
やったこと
C++でのキャプチャ部分実装
WinRT APIを使うためにBuild.csに以下の内容を追加しました。
[プロジェクト名].Build.cs
if (Target.Platform == UnrealTargetPlatform.Win64)
{
bEnableExceptions = true;
bUseUnity = false;
CppStandard = CppStandardVersion.Cpp17;
PublicSystemLibraries.AddRange(new string[] { "shlwapi.lib", "runtimeobject.lib", "D3D11.lib" });
PrivateIncludePaths.Add(System.IO.Path.Combine(Target.WindowsPlatform.WindowsSdkDir,
"Include",
Target.WindowsPlatform.WindowsSdkVersion,
"cppwinrt"));
}
キャプチャのコアクラスを実装、中身の詳細は上記の参照サイトを参照して下さい。
m_WinrtSizeに設定される値とUTexture2Dで作成時に指定する値が食い違っていたのでそれに合わせて実装しています。
WindowCapturer.h
#pragma once
#include "CoreMinimal.h"
#include "UObject/NoExportTypes.h"
#include "Engine/Texture2D.h"
#if PLATFORM_WINDOWS
#include "Windows/AllowWindowsPlatformTypes.h"
#include "Windows/AllowWindowsPlatformAtomics.h"
#include "Windows/PreWindowsApi.h"
#include <unknwn.h>
#include <winrt/base.h>
#include <winrt/Windows.Foundation.h>
#include <winrt/Windows.Foundation.Collections.h>
#include <winrt/Windows.Perception.Spatial.h>
#include <winrt/Windows.System.h>
#include <winrt/Windows.Graphics.h>
#include <winrt/Windows.Graphics.Capture.h>
#include <winrt/Windows.Graphics.DirectX.h>
#include <winrt/Windows.Graphics.DirectX.Direct3D11.h>
#include <d3d11.h>
#include <windows.graphics.capture.interop.h>
#include <windows.graphics.directx.direct3d11.interop.h>
#include "Windows/PostWindowsApi.h"
#include "Windows/HideWindowsPlatformAtomics.h"
#include "Windows/HideWindowsPlatformTypes.h"
#endif
#include "WindowCapturer.generated.h"
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FWindowCapturerChangeTexture, UTexture2D*, NewTexture);
/**
*
*/
UCLASS(BlueprintType, Blueprintable)
class CAPWINDOWNEWAPI_API UWindowCapturer : public UObject
{
GENERATED_BODY()
public:
UWindowCapturer();
virtual ~UWindowCapturer();
UFUNCTION(BlueprintCallable)
virtual void Start();
UFUNCTION(BlueprintCallable)
virtual void Close();
protected:
void ReCreateTexture();
#if PLATFORM_WINDOWS
winrt::Windows::Graphics::Capture::GraphicsCaptureItem CreateCaptureItem(HWND hwnd);
winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice CreateDevice();
void OnFrameArrived(winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, winrt::Windows::Foundation::IInspectable const& args);
void UpdateTextureFromID3D11Texture2D(winrt::com_ptr<ID3D11Texture2D> texture);
#endif
public:
UPROPERTY(BlueprintAssignable, Category = SceneCapture)
FWindowCapturerChangeTexture ChangeTexture;
private:
static bool IsInit;
class UTexture2D* TextureTarget;
#if PLATFORM_WINDOWS
HWND m_TargetWindow = nullptr;
winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice m_WinrtDevice = nullptr;
winrt::Windows::Graphics::Capture::GraphicsCaptureItem m_WinrtItem = nullptr;
winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool m_WinrtFramePool = nullptr;
winrt::Windows::Graphics::Capture::GraphicsCaptureSession m_WinrtSession = nullptr;
winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::FrameArrived_revoker m_WinrtRevoker;
winrt::Windows::Graphics::SizeInt32 m_WinrtSize = { 0, 0 };
winrt::com_ptr<ID3D11Texture2D> m_WinrtTexture = nullptr;
UINT m_Width = 0;
UINT m_Height = 0;
#endif
};
WindowCapture.cpp
#include "WindowCapturer.h"
bool UWindowCapturer::IsInit = false;
UWindowCapturer::UWindowCapturer()
{
if (!IsInit)
{
winrt::init_apartment(winrt::apartment_type::single_threaded);
IsInit = true;
}
}
UWindowCapturer::~UWindowCapturer()
{
Close();
}
void UWindowCapturer::Start()
{
#if PLATFORM_WINDOWS
try
{
if (!m_TargetWindow)
{
// アクティブウィンドウを決め打ちで取得
m_TargetWindow = GetActiveWindow();
}
if (!m_TargetWindow)
{
return;
}
m_WinrtItem = CreateCaptureItem(m_TargetWindow);
if (!m_WinrtItem)
{
UE_LOG(LogTemp, Error, TEXT("Failed to CreateCaptureItem()"));
return;
}
m_WinrtDevice = CreateDevice();
if (!m_WinrtDevice)
{
UE_LOG(LogTemp, Error, TEXT("Failed to CreateDevice()"));
return;
}
// StartCapture
m_WinrtFramePool = winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::Create(
m_WinrtDevice,
winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
2,
m_WinrtItem.Size());
m_WinrtRevoker = m_WinrtFramePool.FrameArrived(winrt::auto_revoke, { this, &UWindowCapturer::OnFrameArrived });
m_WinrtSession = m_WinrtFramePool.CreateCaptureSession(m_WinrtItem);
m_WinrtSession.StartCapture();
}
catch (const winrt::hresult_error& e)
{
const int code = e.code();
const winrt::hstring message = e.message();
UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
}
#endif
}
void UWindowCapturer::Close()
{
#if PLATFORM_WINDOWS
if (TextureTarget)
{
TextureTarget->ReleaseResource();
TextureTarget = nullptr;
}
if (m_WinrtFramePool)
{
m_WinrtRevoker.revoke();
}
if (m_WinrtSession)
{
m_WinrtSession.Close();
m_WinrtSession = nullptr;
}
if (m_WinrtFramePool)
{
m_WinrtFramePool.Close();
m_WinrtFramePool = nullptr;
}
m_WinrtDevice = nullptr;
m_WinrtItem = nullptr;
#endif
}
#if PLATFORM_WINDOWS
winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice UWindowCapturer::CreateDevice()
{
try
{
UINT createDeviceFlags = D3D11_CREATE_DEVICE_FLAG::D3D11_CREATE_DEVICE_BGRA_SUPPORT;
winrt::com_ptr<ID3D11Device> d3dDevice;
winrt::check_hresult(D3D11CreateDevice(
nullptr,
D3D_DRIVER_TYPE_HARDWARE,
nullptr,
createDeviceFlags,
nullptr,
0,
D3D11_SDK_VERSION,
d3dDevice.put(),
nullptr,
nullptr));
winrt::com_ptr<IDXGIDevice> dxgiDevice = d3dDevice.as<IDXGIDevice>();
winrt::com_ptr<::IInspectable> device;
winrt::check_hresult(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.get(), device.put()));
return device.as<winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice>();
}
catch (const winrt::hresult_error& e)
{
const int code = e.code();
const winrt::hstring message = e.message();
UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
return nullptr;
}
}
winrt::Windows::Graphics::Capture::GraphicsCaptureItem UWindowCapturer::CreateCaptureItem(HWND hwnd)
{
try
{
winrt::Windows::Foundation::IActivationFactory factory = winrt::get_activation_factory<winrt::Windows::Graphics::Capture::GraphicsCaptureItem>();
winrt::impl::com_ref<::IGraphicsCaptureItemInterop> interop = factory.as<::IGraphicsCaptureItemInterop>();
winrt::Windows::Graphics::Capture::GraphicsCaptureItem item{ nullptr };
winrt::check_hresult(interop->CreateForWindow(m_TargetWindow, winrt::guid_of<ABI::Windows::Graphics::Capture::IGraphicsCaptureItem>(), reinterpret_cast<void**>(winrt::put_abi(item))));
return item;
}
catch (const winrt::hresult_error& e)
{
const int code = e.code();
const winrt::hstring message = e.message();
UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
return nullptr;
}
}
void UWindowCapturer::OnFrameArrived(winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, winrt::Windows::Foundation::IInspectable const& args)
{
try
{
winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame frame = sender.TryGetNextFrame();
if (!frame)
{
UE_LOG(LogTemp, Warning, TEXT("Failed to TryGetNextFrame() on OnFrameArrived"));
return;
}
winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DSurface surface = frame.Surface();
if (!surface)
{
UE_LOG(LogTemp, Warning, TEXT("Failed to Surface() on OnFrameArrived"));
return;
}
winrt::impl::com_ref<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> access =
surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>();
winrt::check_hresult(access->GetInterface(winrt::guid_of<ID3D11Texture2D>(), m_WinrtTexture.put_void()));
winrt::Windows::Graphics::SizeInt32 Size = frame.ContentSize();
if ((m_WinrtSize.Height != Size.Height) || (m_WinrtSize.Width != Size.Width))
{
m_WinrtSize = Size;
m_WinrtFramePool.Recreate(
m_WinrtDevice,
winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
2,
m_WinrtSize);
}
UpdateTextureFromID3D11Texture2D(m_WinrtTexture);
ChangeTexture.Broadcast(TextureTarget);
}
catch (const winrt::hresult_error& e)
{
const int code = e.code();
const winrt::hstring message = e.message();
UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
}
}
void UWindowCapturer::UpdateTextureFromID3D11Texture2D(winrt::com_ptr<ID3D11Texture2D> Texture)
{
try
{
// First verify that we can map the texture
D3D11_TEXTURE2D_DESC desc;
Texture->GetDesc(&desc);
// Get the device context
winrt::com_ptr<ID3D11Device> d3dDevice;
Texture->GetDevice(d3dDevice.put());
winrt::com_ptr<ID3D11DeviceContext> d3dContext;
d3dDevice->GetImmediateContext(d3dContext.put());
// map the texture
winrt::com_ptr<ID3D11Texture2D> mappedTexture;
D3D11_MAPPED_SUBRESOURCE mapInfo;
mapInfo.RowPitch;
HRESULT hr = d3dContext->Map(
Texture.get(),
0, // Subresource
D3D11_MAP_READ,
0, // MapFlags
&mapInfo);
if (FAILED(hr))
{
if (hr == E_INVALIDARG)
{
D3D11_TEXTURE2D_DESC desc2;
desc2.Width = desc.Width;
desc2.Height = desc.Height;
desc2.MipLevels = desc.MipLevels;
desc2.ArraySize = desc.ArraySize;
desc2.Format = desc.Format;
desc2.SampleDesc = desc.SampleDesc;
desc2.Usage = D3D11_USAGE_STAGING;
desc2.BindFlags = 0;
desc2.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
desc2.MiscFlags = 0;
winrt::com_ptr<ID3D11Texture2D> stagingTexture;
winrt::check_hresult(d3dDevice->CreateTexture2D(&desc2, nullptr, stagingTexture.put()));
// copy the texture to a staging resource
d3dContext->CopyResource(stagingTexture.get(), Texture.get());
// now, map the staging resource
winrt::check_hresult(d3dContext->Map(
stagingTexture.get(),
0,
D3D11_MAP_READ,
0,
&mapInfo));
mappedTexture = std::move(stagingTexture);
}
else
{
winrt::check_hresult(hr);
}
}
else
{
mappedTexture = Texture;
}
if (!m_WinrtTexture) return;
UINT Width = desc.Width;
UINT Height = desc.Height;
if ((m_Width != Width) ||
(m_Height != Height))
{
m_Width = Width;
m_Height = Height;
ReCreateTexture();
}
auto Region = new FUpdateTextureRegion2D(0, 0, 0, 0, m_Width, m_Height);
TextureTarget->UpdateTextureRegions(0, 1, Region, mapInfo.RowPitch, 4, (uint8*)mapInfo.pData);
}
catch (const winrt::hresult_error& e)
{
const int code = e.code();
const winrt::hstring message = e.message();
UE_LOG(LogTemp, Error, TEXT("winrt::hresult_error %d %s"), code, message.c_str());
}
}
#endif
void UWindowCapturer::ReCreateTexture()
{
#if PLATFORM_WINDOWS
if (m_Height == 0 || m_Width == 0)
{
TextureTarget = nullptr;
return;
}
TextureTarget = UTexture2D::CreateTransient(m_Width, m_Height, PF_B8G8R8A8);
TextureTarget->UpdateResource();
#endif
}
BlueprintでのC++クラスの利用
C++で実装したクラスを使ってStaticMeshでPlaneを持つアクタでテクスチャを表示しています。
WindowCapture2DプラグインのWinRT API版
WindowCapture2DプラグインをForkしてWinRT APIを使ってウィンドウキャプチャするように改造してみました。
(GDI版は古いWindowsもサポートしている&今のやり方だとUE4.26以降しかサポートしていないと思うのでpullreqはしていません、切り替えられたらいいのかな？)
https://github.com/mechamogera/WindowCapture2D
対象ウィンドウが閉じられたことを検知する
以下のようにwinrt::Windows::Graphics::Capture::GraphicsCaptureItem::Closedのイベントによって対象ウィンドウが閉じられたことを検知できるみたいです。
WindowCapture.cpp
{
...
m_WinrtItem.Closed({ this, &UWindowCapturer::OnTargetClosed });
...
}
...
void UWindowCapturer::OnTargetClosed(winrt::Windows::Graphics::Capture::GraphicsCaptureItem Item, winrt::Windows::Foundation::IInspectable Inspectable)
{
フレームレートを指定してキャプチャする
FrameArrivedを利用せず以下のようにすることで指定したフレームレートでキャプチャできるみたいです。
WindowCapture.h
#include "Containers/Ticker.h"
...
UCLASS(BlueprintType, Blueprintable)
class CAPWINDOWNEWAPI_API UWindowCapturer : public UObject
{
...
bool Tick(float deltaTime);
...
FTickerDelegate TickDelegate;
FDelegateHandle TickHandle;
...
}
WindowCapture.cpp
void UWindowCapturer::Start()
{
...
if (TickHandle.IsValid())
{
FTicker::GetCoreTicker().RemoveTicker(TickHandle);
TickHandle.Reset();
}
TickDelegate = FTickerDelegate::CreateUObject(this, &UWindowCapturer::Tick);
TickHandle = FTicker::GetCoreTicker().AddTicker(TickDelegate, 1.0f / 10.0f;
...
}
void UWindowCapturer::Close()
{
#if PLATFORM_WINDOWS
if (TickHandle.IsValid())
{
FTicker::GetCoreTicker().RemoveTicker(TickHandle);
TickHandle.Reset();
}
...
}
...
bool UWindowCapturer::Tick(float deltaTime)
{
OnFrameArrived(m_WinrtFramePool, nullptr);
return true;
}