【SwiftUI】最低限のコードでカメラのプレビューをMetalで表示する

SwiftUI上で必要最低限のコードでカメラのプレビューをMetalで表示します。

• フロントカメラのみ
• デバイスの向きはポートレートのみ
• カメラのアクセス許可のチェックなし

環境

• Swift: 5.4
• Xcode: 12.5 (12E262)
• macOS: Big Sur 11.3.1 (20E241)

コード

ContentView.swift

import SwiftUI
import AVFoundation
import MetalKit

struct ContentView: View {
    var body: some View {
        CameraView()
            .edgesIgnoringSafeArea(.all)
    }
}

struct CameraView: UIViewRepresentable {
    func makeUIView(context: Context) -> some UIView { BaseCameraView() }
    func updateUIView(_ uiView: UIViewType, context: Context) {}
}

class BaseCameraView: UIView, AVCaptureVideoDataOutputSampleBufferDelegate {
    let metalLayer = CAMetalLayer()
    let device = MTLCreateSystemDefaultDevice()!
    lazy var commandQueue = device.makeCommandQueue()
    let renderPassDescriptor = MTLRenderPassDescriptor()
    lazy var renderPipelineState: MTLRenderPipelineState! = {
        guard let library = device.makeDefaultLibrary() else { return nil }
        let descriptor = MTLRenderPipelineDescriptor()
        descriptor.vertexFunction = library.makeFunction(name: "vertexShader")
        descriptor.fragmentFunction = library.makeFunction(name: "fragmentShader")
        descriptor.colorAttachments[0].pixelFormat = .bgra8Unorm
        return try? device.makeRenderPipelineState(descriptor: descriptor)
    }()
    let captureSession = AVCaptureSession()
    var lockFlag = false

    override func layoutSubviews() {
        super.layoutSubviews()
        _ = initMetalAndCaptureSession
        metalLayer.frame = layer.frame
    }

    lazy var initMetalAndCaptureSession: Void = {
        metalLayer.device = device
        metalLayer.isOpaque = false
        layer.addSublayer(metalLayer)

        renderPassDescriptor.colorAttachments[0].loadAction = .clear
        renderPassDescriptor.colorAttachments[0].storeAction = .store
        renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0)

        guard let captureDevice = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera],
                                                                   mediaType: .video,
                                                                   position: .front).devices.first,
              let input = try? AVCaptureDeviceInput(device: captureDevice) else { return }

        let output = AVCaptureVideoDataOutput()
        output.videoSettings = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA] as [String : Any]
        output.setSampleBufferDelegate(self, queue: DispatchQueue.main)

        captureSession.addInput(input)
        captureSession.addOutput(output)
        captureSession.startRunning()
        captureSession.connections.forEach {
            $0.videoOrientation = .portrait
            $0.isVideoMirrored = true
        }
    }()

    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        if lockFlag { return }
        lockFlag = true

        guard let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
        CVPixelBufferLockBaseAddress(buffer, .readOnly)

        let width = CVPixelBufferGetWidth(buffer)
        let height = CVPixelBufferGetHeight(buffer)

        var textureCache: CVMetalTextureCache!
        CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &textureCache)
        var texture: CVMetalTexture!
        _ = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, buffer, nil, .bgra8Unorm, width, height, 0, &texture)

        guard let drawable = metalLayer.nextDrawable(),
              let commandBuffer = commandQueue?.makeCommandBuffer() else { return }

        renderPassDescriptor.colorAttachments[0].texture = drawable.texture
        guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return }
        encoder.setRenderPipelineState(renderPipelineState)

        let aspect = Float(frame.width / frame.height) * Float(height) / Float(width)
        let vertexData: [[Float]] = [
            // 0: positions
            [
                -1, -aspect, 0, 1,
                -1, aspect, 0, 1,
                1, -aspect, 0, 1,
                1, aspect, 0, 1,
            ],
            // 1: texCoords
            [
                0, 1,
                0, 0,
                1, 1,
                1, 0,
            ],
        ]

        vertexData.enumerated().forEach { i, array in
            let size = array.count * MemoryLayout.size(ofValue: array[0])
            let buffer = device.makeBuffer(bytes: array, length: size)
            encoder.setVertexBuffer(buffer, offset: 0, index: i)
        }

        encoder.setFragmentTexture(CVMetalTextureGetTexture(texture), index: 0)
        encoder.drawPrimitives(type: .triangleStrip,
                               vertexStart: 0,
                               vertexCount: vertexData[0].count / 4)

        encoder.endEncoding()
        commandBuffer.present(drawable)
        commandBuffer.commit()

        CVPixelBufferUnlockBaseAddress(buffer, .readOnly)
        lockFlag = false
    }
}

struct ContentView_Previews: PreviewProvider {
    static var previews: some View {
        ContentView()
    }
}

Shader.metal

#include <metal_stdlib>
using namespace metal;

struct RasterizerData
{
    float4 clipSpacePosition [[position]];
    float2 texCoord;
};

vertex RasterizerData
vertexShader(uint vertexID [[ vertex_id ]],
             const device float4 *position [[ buffer(0) ]],
             const device float2 *uv [[ buffer(1) ]])
{
    RasterizerData out;
    out.clipSpacePosition = position[vertexID];
    out.texCoord = uv[vertexID];
    return out;
}

fragment float4
fragmentShader(RasterizerData in [[ stage_in ]],
               texture2d<float, access::sample> texture [[ texture(0) ]])
{
    constexpr sampler sampler2d(coord::normalized, filter::linear, address::clamp_to_zero);
    return texture.sample(sampler2d, in.texCoord);
}

Info.plist

    <key>NSCameraUsageDescription</key>
    <string>This app uses the camera</string>

リポジトリ