HoloLens アドベントカレンダー24日目の記事です！
HoloLensのタップ操作により、写真を撮ると指が映りこんでしまい、取り直すことがよくあります。今回は、HoloLensのカメラの画角を3Dマップ上に可視化してみました。
開発環境
- Unity 2017.1.2f1
- Visual Studio 2017(15.4.4)
- HoloToolkit-Unity-v1.2017.2.0.unitypackage
- HoloPointCloud
- Point Cloud Free Viewer
実装
こちらの記事HoloLensで部屋の3Dスキャン試してみたをベースに作っていきます。
HoloPointCloudをダウンロードしてください。
Unityで開き、Projectビューのsaco->Scenes->PhotoCaptureRawシーンを選択します。
HoloToolkitとPoint Cloud Free ViewerのShaderフォルダのみインポートします。
SpatialMappingのInspectorビューにあるPhotoCaptureRaw.csを次のように編集します。
using HoloToolkit.Unity.InputModule;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
using UnityEngine.VR.WSA.WebCam;
public class PhotoCaptureRaw : MonoBehaviour, IInputClickHandler
{
private PhotoCapture photoCaptureObject = null;
private bool isCapturing = false;
private int Width;
private int Height;
public int lengthOfLineRenderer;
// Use this for initialization
void Start()
{
InputManager.Instance.PushFallbackInputHandler(gameObject);
}
// Update is called once per frame
void Update()
{
//if (Input.GetKeyDown("space"))
//{
// FrameDisplay();
//}
}
void /*FrameDisplay()*/IInputClickHandler.OnInputClicked(InputClickedEventData eventData)
{
Debug.Log("Tapped");
if (!isCapturing)
{
if (photoCaptureObject != null)
{
photoCaptureObject.Dispose();
photoCaptureObject = null;
}
PhotoCapture.CreateAsync(false, OnPhotoCaptureCreated);
isCapturing = true;
}
}
void OnPhotoCaptureCreated(PhotoCapture captureObject)
{
photoCaptureObject = captureObject;
CameraParameters c = new CameraParameters();
Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
c.hologramOpacity = 0.0f;
c.cameraResolutionWidth = cameraResolution.width;
c.cameraResolutionHeight = cameraResolution.height;
c.pixelFormat = CapturePixelFormat.BGRA32;
Width = cameraResolution.width;
Height = cameraResolution.height;
Debug.Log(String.Format("width={0}, height={1}", Width, Height));
//captureObject.StartPhotoModeAsync(c, OnPhotoModeStarted);
// Activate the camera
photoCaptureObject.StartPhotoModeAsync(c, delegate (PhotoCapture.PhotoCaptureResult result) {
// Take a picture
photoCaptureObject.TakePhotoAsync(OnCapturedPhotoToMemory);
});
}
void CreateFrame(PhotoCaptureFrame photoCaptureFrame)
{
// ------------------------------------------------------
// make xyxrgb
//
int texWidth = Width / 8;
int texHeight = Height / 8;
int numPoints = texWidth * texHeight;
List<Vector3> points = new List<Vector3>();
List<int> indecies = new List<int>();
// ------------------------------------------------------
// pos & rot
//
Matrix4x4 cameraToWorldMatrix;
photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
Matrix4x4 projectionMatrix;
photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix);
// 0 right
// 1 up
// 2 forward
// 3 position
// Position the canvas object slightly in front
// of the real world web camera.
#if UNITY_EDITOR
Vector3 position = cameraToWorldMatrix.GetColumn(3) + cameraToWorldMatrix.GetColumn(2);
#else
Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
#endif
// Rotate the canvas object so that it faces the user.
Quaternion rotation = Quaternion.LookRotation(cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
// ------------------------------------------------------
// pos -> ray
//
int pointCount = 0;
{
int y = texHeight - 1;
for (int x = texWidth - 1; x >= 0; --x)
{
//int i = y * texWidth + x;
points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
(y - (texHeight / 2)) / 200.0F, 0.0F));
#if false
Vector3 point = points[pointCount];
#else
// SetTRS
Vector3 translation = position;
//Vector3 eulerAngles;
Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
Matrix4x4 m = Matrix4x4.identity;
//Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);
m.SetTRS(translation, rotation, scale);
Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif
Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
point.y - cameraToWorldMatrix.GetColumn(3).y,
point.z - cameraToWorldMatrix.GetColumn(3).z);
RaycastHit hit;
if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
{
points[pointCount] = hit.point;
indecies.Add(pointCount);
pointCount++;
}
else
{
points.RemoveAt(pointCount);
}
}
}
{
int x = 0;
for (int y = texHeight - 2; y >= 1; --y)
{
points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
(y - (texHeight / 2)) / 200.0F, 0.0F));
//int i = y * texWidth + x;
#if false
Vector3 point = points[pointCount];
#else
// SetTRS
Vector3 translation = position;
//Vector3 eulerAngles;
Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
Matrix4x4 m = Matrix4x4.identity;
//Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);
m.SetTRS(translation, rotation, scale);
Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif
Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
point.y - cameraToWorldMatrix.GetColumn(3).y,
point.z - cameraToWorldMatrix.GetColumn(3).z);
RaycastHit hit;
if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
{
points[pointCount] = hit.point;
indecies.Add(pointCount);
pointCount++;
}
else
{
points.RemoveAt(pointCount);
}
}
}
{
int y = 0;
for (int x = 0; x <= texWidth - 1; ++x)
{
//int i = y * texWidth + x;
points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
(y - (texHeight / 2)) / 200.0F, 0.0F));
#if false
Vector3 point = points[pointCount];
#else
// SetTRS
Vector3 translation = position;
//Vector3 eulerAngles;
Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
Matrix4x4 m = Matrix4x4.identity;
//Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);
m.SetTRS(translation, rotation, scale);
Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif
Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
point.y - cameraToWorldMatrix.GetColumn(3).y,
point.z - cameraToWorldMatrix.GetColumn(3).z);
RaycastHit hit;
if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
{
points[pointCount] = hit.point;
indecies.Add(pointCount);
pointCount++;
}
else
{
points.RemoveAt(pointCount);
}
}
}
{
int x = texWidth - 1;
for (int y = 1; y < texHeight - 1; ++y)
{
//int i = y * texWidth + x;
points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
(y - (texHeight / 2)) / 200.0F, 0.0F));
#if false
Vector3 point = points[pointCount];
#else
// SetTRS
Vector3 translation = position;
//Vector3 eulerAngles;
Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
Matrix4x4 m = Matrix4x4.identity;
//Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);
m.SetTRS(translation, rotation, scale);
Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif
Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
point.y - cameraToWorldMatrix.GetColumn(3).y,
point.z - cameraToWorldMatrix.GetColumn(3).z);
RaycastHit hit;
if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
{
points[pointCount] = hit.point;
indecies.Add(pointCount);
pointCount++;
}
else
{
points.RemoveAt(pointCount);
}
}
}
Debug.Log(pointCount);
LineRenderer lineRenderer = GetComponent<LineRenderer>();
lengthOfLineRenderer = pointCount;
lineRenderer.positionCount = pointCount;
var t = Time.time;
for (int i = 0; i < lengthOfLineRenderer; i++)
{
lineRenderer.SetPosition(i, points[i]);
}
}
void OnStoppedPhotoMode(PhotoCapture.PhotoCaptureResult result)
{
photoCaptureObject.Dispose();
photoCaptureObject = null;
isCapturing = false;
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
string filename = string.Format(@"CapturedImage{0}_n.jpg", Time.time);
string filePath = System.IO.Path.Combine(Application.persistentDataPath, filename);
photoCaptureObject.TakePhotoAsync(filePath, PhotoCaptureFileOutputFormat.JPG, OnCapturedPhotoToDisk);
CreateFrame(photoCaptureFrame);
}
else
{
Debug.LogError("Unable to start photo mode!");
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
isCapturing = false;
}
void OnCapturedPhotoToDisk(PhotoCapture.PhotoCaptureResult result)
{
if (result.success)
{
Debug.Log("Saved Photo to disk!");
}
else
{
Debug.Log("Failed to save Photo to disk");
}
}
}
画像枠に対して、RayCastのHitPointをLineRendererでつないで表示しています。
SpatialMappingにLineRendererを追加し、Widthを0.01に設定します。
いつものようにビルドして、実行してください。
タップすると撮影された範囲の画角が表示されます。
実行動画はこちらです。
指が映りこんでいたら音などで知らせてくれるといいかもです。
PhotoCaptureFrameを取得してからでないとカメラのMatrixが得られないので、常に撮影する必要がありますね...
Fig.1 MRCで撮影した画像
Fig.2 PhotoCaptureで撮影した画像