骨格検出アプリ作成２
前回、動画の呼び出しまでできたのでその読み込んだ動画に対して骨格検出を行いたいと思います。
できたもの
仕組み
1.読み込んだ動画(a)をフレーム単位で静止画(b)にする
2.(b)に対して骨格検出を行い、骨格が表示された静止画(c)を作成する
3.(c)をドキュメント内に再生順に保存していく
4.(c)群を並べて動画にする
5.動画にする際の再生レートを(a)を参照して設定する
色々と方法を考えてはみましたが、知識不足と相まって回りくどいやり方になってしまいました。ここに至るまで、至ってからも色々と大変ポイントがあったのですが、とりあえず割愛します。
コード1.読み込んだ動画(a)をフレーム単位で静止画(b)にする
private func trimVideo(){
let trimAsset = (videoPlayer?.currentItem?.asset)!
let composition = AVMutableComposition()
guard let videoTrack = composition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid) else {
debugPrint("Failed to add video track")
return
}
// 素材Assetの1個目のVideoトラックを使う
let srcVideoTrack = trimAsset.tracks(withMediaType: .video)[0]
do {
// CMTimeMakeSecondsはpreferredTimescaleを100とすることで秒数が正確に表せる
try videoTrack.insertTimeRange(CMTimeRangeMake(start: CMTimeMakeWithSeconds(cutStartTime, preferredTimescale: 100), duration: CMTimeMakeWithSeconds(cutEndTime,preferredTimescale: 100)), of: srcVideoTrack, at: .zero)
} catch { print(error) }
self.cutplayerItem = AVPlayerItem(asset: composition)
self.cutvideo = AVPlayer(playerItem: self.cutplayerItem)
// 動画をフレームに直す処理準備
let cutAsset = cutvideo?.currentItem?.asset
let assetReader = try! AVAssetReader(asset: cutAsset!)
let CutvideoTrack = cutAsset!.tracks(withMediaType: .video)[0]
let outputSettings: [String: Any] = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
let trackOutput = AVAssetReaderTrackOutput(track: CutvideoTrack, outputSettings: outputSettings)
assetReader.add(trackOutput)
// 読み込みの開始
assetReader.startReading()
var frameNumber = 0
while let sampleBuffer = trackOutput.copyNextSampleBuffer(){
guard let pixcelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
continue
}
if CMSampleBufferGetImageBuffer(sampleBuffer) != nil {
let ciImage = CIImage(cvPixelBuffer: pixcelBuffer)
let context = CIContext(options: nil)
let cgImage = context.createCGImage(ciImage, from: ciImage.extent)!
let image = UIImage(cgImage: cgImage)
// 骨格検出関数 image:取り出したフレーム,frameNumber:ファイル名保存用の変数
self.poseDetect(image: image, frameNumber: frameNumber)
frameNumber += 1
} else {
break
}
}
}
コード2.(b)に対して骨格検出を行い、骨格が表示された静止画(c)を作成する
// 人物の関節点と関節点同士を結んだ骨格線を予測する
func poseDetect(image:UIImage, frameNumber:Int){
// 画像に直接合成して保存する場合は画像のサイズを参照する
let windowSize_w = image.size.width
let windowSize_h = image.size.height
// 処理したい画像（フレーム）を保持する関数
let imageRequestHandler = VNImageRequestHandler(cgImage:image.cgImage!)
// 画像中の人物の骨格情報を推定するもの（モデル含む？）
let poseEstimationRequest = VNDetectHumanBodyPoseRequest()
do {
// poseEstimationRequestを使ってimageRequestHandlerの画像を解析する
try imageRequestHandler.perform([poseEstimationRequest])
} catch {
print("error")
}
guard let results = poseEstimationRequest.results as? [VNHumanBodyPoseObservation] else{return}
do {
for obserbation in results {
// 右腕
let rightShoulder = try obserbation.recognizedPoint(.rightShoulder).location
let rightElbow = try obserbation.recognizedPoint(.rightElbow).location
let rightWrist = try obserbation.recognizedPoint(.rightWrist).location
// 左腕
let leftShoulder = try obserbation.recognizedPoint(.leftShoulder).location
let leftElbow = try obserbation.recognizedPoint(.leftElbow).location
let leftWrist = try obserbation.recognizedPoint(.leftWrist).location
// 腰
let root = try obserbation.recognizedPoint(.root).location
// 右脚
let rightHip = try obserbation.recognizedPoint(.rightHip).location
let rightKnee = try obserbation.recognizedPoint(.rightKnee).location
let rightAnkle = try obserbation.recognizedPoint(.rightAnkle).location
// 左脚
let leftHip = try obserbation.recognizedPoint(.leftHip).location
let leftKnee = try obserbation.recognizedPoint(.leftKnee).location
let leftAnkle = try obserbation.recognizedPoint(.leftAnkle).location
let RS_x = rightShoulder.x * windowSize_w
let RS_y = (1 - rightShoulder.y) * windowSize_h
let RS_cir = UIBezierPath(arcCenter: CGPoint(x: RS_x, y: RS_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let RS_lay = CAShapeLayer()
RS_lay.path = RS_cir.cgPath
RS_lay.fillColor = UIColor.red.cgColor
RS_lay.lineWidth = 3.0
let RE_x = rightElbow.x * windowSize_w
let RE_y = (1 - rightElbow.y) * windowSize_h
let RE_cir = UIBezierPath(arcCenter: CGPoint(x: RE_x, y: RE_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let RE_lay = CAShapeLayer()
RE_lay.path = RE_cir.cgPath
RE_lay.fillColor = UIColor.red.cgColor
RE_lay.lineWidth = 3.0
let RW_x = rightWrist.x * windowSize_w
let RW_y = (1 - rightWrist.y) * windowSize_h
let RW_cir = UIBezierPath(arcCenter: CGPoint(x: RW_x, y: RW_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let RW_lay = CAShapeLayer()
RW_lay.path = RW_cir.cgPath
RW_lay.fillColor = UIColor.red.cgColor
RW_lay.lineWidth = 3.0
let LS_x = leftShoulder.x * windowSize_w
let LS_y = (1 - leftShoulder.y) * windowSize_h
let LS_cir = UIBezierPath(arcCenter: CGPoint(x: LS_x, y: LS_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let LS_lay = CAShapeLayer()
LS_lay.path = LS_cir.cgPath
LS_lay.fillColor = UIColor.red.cgColor
LS_lay.lineWidth = 3.0
let LE_x = leftElbow.x * windowSize_w
let LE_y = (1 - leftElbow.y) * windowSize_h
let LE_cir = UIBezierPath(arcCenter: CGPoint(x: LE_x, y: LE_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let LE_lay = CAShapeLayer()
LE_lay.path = LE_cir.cgPath
LE_lay.fillColor = UIColor.red.cgColor
LE_lay.lineWidth = 3.0
let LW_x = leftWrist.x * windowSize_w
let LW_y = (1 - leftWrist.y) * windowSize_h
let LW_cir = UIBezierPath(arcCenter: CGPoint(x: LW_x, y: LW_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let LW_lay = CAShapeLayer()
LW_lay.path = LW_cir.cgPath
LW_lay.fillColor = UIColor.red.cgColor
LW_lay.lineWidth = 3.0
let RO_x = root.x * windowSize_w
let RO_y = (1 - root.y) * windowSize_h
let RO_cir = UIBezierPath(arcCenter: CGPoint(x: RO_x, y: RO_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let RO_lay = CAShapeLayer()
RO_lay.path = RO_cir.cgPath
RO_lay.fillColor = UIColor.red.cgColor
RO_lay.lineWidth = 3.0
let RH_x = rightHip.x * windowSize_w
let RH_y = (1 - rightHip.y) * windowSize_h
let RH_cir = UIBezierPath(arcCenter: CGPoint(x: RH_x, y: RH_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let RH_lay = CAShapeLayer()
RH_lay.path = RH_cir.cgPath
RH_lay.fillColor = UIColor.red.cgColor
RH_lay.lineWidth = 3.0
let RK_x = rightKnee.x * windowSize_w
let RK_y = (1 - rightKnee.y) * windowSize_h
let RK_cir = UIBezierPath(arcCenter: CGPoint(x: RK_x, y: RK_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let RK_lay = CAShapeLayer()
RK_lay.path = RK_cir.cgPath
RK_lay.fillColor = UIColor.red.cgColor
RK_lay.lineWidth = 3.0
let RA_x = rightAnkle.x * windowSize_w
let RA_y = (1 - rightAnkle.y) * windowSize_h
let RA_cir = UIBezierPath(arcCenter: CGPoint(x: RA_x, y: RA_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let RA_lay = CAShapeLayer()
RA_lay.path = RA_cir.cgPath
RA_lay.fillColor = UIColor.red.cgColor
RA_lay.lineWidth = 3.0
let LH_x = leftHip.x * windowSize_w
let LH_y = (1 - leftHip.y) * windowSize_h
let LH_cir = UIBezierPath(arcCenter: CGPoint(x: LH_x, y: LH_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let LH_lay = CAShapeLayer()
LH_lay.path = LH_cir.cgPath
LH_lay.fillColor = UIColor.red.cgColor
LH_lay.lineWidth = 3.0
let LA_x = leftAnkle.x * windowSize_w
let LA_y = (1 - leftAnkle.y) * windowSize_h
let LA_cir = UIBezierPath(arcCenter: CGPoint(x: LA_x, y: LA_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let LA_lay = CAShapeLayer()
LA_lay.path = LA_cir.cgPath
LA_lay.fillColor = UIColor.red.cgColor
LA_lay.lineWidth = 3.0
let LK_x = leftKnee.x * windowSize_w
let LK_y = (1 - leftKnee.y) * windowSize_h
let LK_cir = UIBezierPath(arcCenter: CGPoint(x: LK_x, y: LK_y), radius: 2, startAngle: 0, endAngle: CGFloat(Double.pi * 2.0), clockwise:true)
let LK_lay = CAShapeLayer()
LK_lay.path = LK_cir.cgPath
LK_lay.fillColor = UIColor.red.cgColor
LK_lay.lineWidth = 3.0
// 描画する線をまとめるレイヤー
let layer = CALayer()
// 右肩から右肘
let RStoRE_lay = CAShapeLayer()
RStoRE_lay.lineWidth = 2.0
RStoRE_lay.strokeColor = UIColor.green.cgColor
let path1 = UIBezierPath()
path1.move(to: CGPoint(x: RS_x, y: RS_y))
path1.addLine(to: CGPoint(x: RE_x, y: RE_y))
RStoRE_lay.path = path1.cgPath
layer.addSublayer(RStoRE_lay)
// 右肘から右手首
let REtoRW_lay = CAShapeLayer()
REtoRW_lay.lineWidth = 2.0
REtoRW_lay.strokeColor = UIColor.green.cgColor
let path2 = UIBezierPath()
path2.move(to: CGPoint(x: RE_x, y: RE_y))
path2.addLine(to: CGPoint(x: RW_x, y: RW_y))
REtoRW_lay.path = path2.cgPath
layer.addSublayer(REtoRW_lay)
// 右肩から左肩
let RStoLS_lay = CAShapeLayer()
RStoLS_lay.lineWidth = 2.0
RStoLS_lay.strokeColor = UIColor.green.cgColor
let path3 = UIBezierPath()
path3.move(to: CGPoint(x: RS_x, y: RS_y))
path3.addLine(to: CGPoint(x: LS_x, y: LS_y))
RStoLS_lay.path = path3.cgPath
layer.addSublayer(RStoLS_lay)
// 左肩から左肘
let LStoLE_lay = CAShapeLayer()
LStoLE_lay.lineWidth = 2.0
LStoLE_lay.strokeColor = UIColor.green.cgColor
let path4 = UIBezierPath()
path4.move(to: CGPoint(x: LS_x, y: LS_y))
path4.addLine(to: CGPoint(x: LE_x, y: LE_y))
LStoLE_lay.path = path4.cgPath
layer.addSublayer(LStoLE_lay)
// 左肘から左手首
let LEtoLW_lay = CAShapeLayer()
LEtoLW_lay.lineWidth = 2.0
LEtoLW_lay.strokeColor = UIColor.green.cgColor
let path5 = UIBezierPath()
path5.move(to: CGPoint(x: LE_x, y: LE_y))
path5.addLine(to: CGPoint(x: LW_x, y: LW_y))
LEtoLW_lay.path = path5.cgPath
layer.addSublayer(LEtoLW_lay)
// 左肩から左尻
let LStoLH_lay = CAShapeLayer()
LStoLH_lay.lineWidth = 2.0
LStoLH_lay.strokeColor = UIColor.green.cgColor
let path6 = UIBezierPath()
path6.move(to: CGPoint(x: LS_x, y: LS_y))
path6.addLine(to: CGPoint(x: LH_x, y: LH_y))
LStoLH_lay.path = path6.cgPath
layer.addSublayer(LStoLH_lay)
// 右肩から右尻
let RStoRH_lay = CAShapeLayer()
RStoRH_lay.lineWidth = 2.0
RStoRH_lay.strokeColor = UIColor.green.cgColor
let path7 = UIBezierPath()
path7.move(to: CGPoint(x: RS_x, y: RS_y))
path7.addLine(to: CGPoint(x: RH_x, y: RH_y))
RStoRH_lay.path = path7.cgPath
layer.addSublayer(RStoRH_lay)
// 右尻から右膝
let RKtoRH_lay = CAShapeLayer()
RKtoRH_lay.lineWidth = 2.0
RKtoRH_lay.strokeColor = UIColor.green.cgColor
let path8 = UIBezierPath()
path8.move(to: CGPoint(x: RK_x, y: RK_y))
path8.addLine(to: CGPoint(x: RH_x, y: RH_y))
RKtoRH_lay.path = path8.cgPath
layer.addSublayer(RKtoRH_lay)
// 左尻から左膝
let LKtoLH_lay = CAShapeLayer()
LKtoLH_lay.lineWidth = 2.0
LKtoLH_lay.strokeColor = UIColor.green.cgColor
let path9 = UIBezierPath()
path9.move(to: CGPoint(x: LK_x, y: LK_y))
path9.addLine(to: CGPoint(x: LH_x, y: LH_y))
LKtoLH_lay.path = path9.cgPath
layer.addSublayer(LKtoLH_lay)
// 右膝から右足首
let RKtoRA_lay = CAShapeLayer()
RKtoRA_lay.lineWidth = 2.0
RKtoRA_lay.strokeColor = UIColor.green.cgColor
let path10 = UIBezierPath()
path10.move(to: CGPoint(x: RK_x, y: RK_y))
path10.addLine(to: CGPoint(x: RA_x, y: RA_y))
RKtoRA_lay.path = path10.cgPath
layer.addSublayer(RKtoRA_lay)
// 左膝から左足首
let LKtoLA_lay = CAShapeLayer()
LKtoLA_lay.lineWidth = 2.0
LKtoLA_lay.strokeColor = UIColor.green.cgColor
let path11 = UIBezierPath()
path11.move(to: CGPoint(x: LK_x, y: LK_y))
path11.addLine(to: CGPoint(x: LA_x, y: LA_y))
LKtoLA_lay.path = path11.cgPath
layer.addSublayer(LKtoLA_lay)
// 右尻から左尻
let RHtoLH_lay = CAShapeLayer()
RHtoLH_lay.lineWidth = 2.0
RHtoLH_lay.strokeColor = UIColor.green.cgColor
let path12 = UIBezierPath()
path12.move(to: CGPoint(x: RH_x, y: RH_y))
path12.addLine(to: CGPoint(x: LH_x, y: LH_y))
RHtoLH_lay.path = path12.cgPath
layer.addSublayer(RHtoLH_lay)
// 判定した画像に対して線を描画する
let resultImage = drawShapeOnImage(image: image, shapeLayer: layer)
//線付きの人物画像をアプリ内に保存していく 本来は画像を並べて動画に直す作業に持っていく
let imageData = resultImage!.jpegData(compressionQuality: 1.0)
let documentURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
let fileURL = documentURL.appendingPathComponent("frame\(frameNumber).jpg")
try? imageData?.write(to: fileURL)
}
}catch{
print("error")
}
}
ループ等を使えばもっと綺麗に描けるのかなと思いつつ、直書きでお恥ずかしいです。
3.4.5
// 動画を生成するファンクション まず構造体を生成
func createVideo(fps:Float) {
struct dirImage {
var URL: URL
var name: String
var image: UIImage
init(URL: URL, name: String, image: UIImage){
self.URL = URL
self.name = name
self.image = image
}
}
// 画像を読み込み、配列に入れる
var imagesArray = [dirImage]()
let fileManager = FileManager.default
let documentsUrl = fileManager.urls(for: .documentDirectory, in: .userDomainMask)[0]
let fileUrls = try! fileManager.contentsOfDirectory(at: documentsUrl, includingPropertiesForKeys: nil, options: [])
for fileUrl in fileUrls {
if let Frameimage = UIImage(contentsOfFile: fileUrl.path){
imagesArray.append(dirImage(URL:fileUrl, name: fileUrl.lastPathComponent, image: Frameimage))
}
}
// 名前順に入れ替え
imagesArray = imagesArray.sorted { (name1, name2) -> Bool in
let numericPart1 = name1.name.replacingOccurrences(of: "[^0-9]", with: "",options: .regularExpression)
let numericPart2 = name2.name.replacingOccurrences(of: "[^0-9]", with: "",options: .regularExpression)
guard let number1 = Int(numericPart1), let number2 = Int(numericPart2) else {
return name1.name < name2.name
}
return number1 < number2
}
print(imagesArray)
// 生成した動画を保存するパス
let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0]
as String
let videoOutputPath = (documentsPath as NSString).appendingPathComponent("output.mp4")
let videoWriter = try! AVAssetWriter(outputURL: URL(fileURLWithPath: videoOutputPath), fileType: .mp4)
// 既にファイルがある場合は削除する
// このコードを消したりつけたりする必要がある点を修正
if fileManager.fileExists(atPath: documentsPath) {
try! fileManager.removeItem(atPath: videoOutputPath)
}
// 最初の画像から動画のサイズを取得する
let size = imagesArray.first?.image.size
// 動画の保存先の指定、保存方法、動画のフレームを書き込むための準備
guard let videoWriter = try? AVAssetWriter(outputURL: URL(fileURLWithPath: videoOutputPath), fileType: .mp4) else {
abort()
}
let videoSettings = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: size?.width,
AVVideoHeightKey: size?.height
] as [String : Any]
let videoWriterInput = AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: videoSettings)
videoWriterInput.expectsMediaDataInRealTime = true
videoWriter.add(videoWriterInput)
// 画像をフレームとして書き込むための用意
let sourcePixcelBufferAttributesDictionary = [
kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32ARGB),
kCVPixelBufferWidthKey as String: size?.width,
kCVPixelBufferHeightKey as String: size?.height
] as [String : Any]
let adaptor = AVAssetWriterInputPixelBufferAdaptor(
assetWriterInput: videoWriterInput,
sourcePixelBufferAttributes: sourcePixcelBufferAttributesDictionary)
videoWriterInput.expectsMediaDataInRealTime = true
// 動画生成開始
if (!videoWriter.startWriting()){
print("Failed to start writing")
return
}
videoWriter.startSession(atSourceTime: CMTime.zero)
var frameCount: Int64 = 0
// 各画像の表示する時間
let durationForEachImage: Int64 = 1
//let durationForEachImage: Float = 0.1
//let fps: Int32 = 24
let fps = fps
// 構造体から要素を取り出す [name, uiimage]
for img in imagesArray {
if !adaptor.assetWriterInput.isReadyForMoreMediaData {
continue
}
// 取り出した要素の画像を選択しそれをcgImageに変換
let cgImg = img.image.cgImage
// 動画の時間を生成（その画像を表示する時間/開始時間と表示時間を渡す）
//let frameTime = CMTimeMake(value: frameCount * Int64(fps) * durationForEachImage,timescale: fps)
let frameTime = CMTimeMake(value: frameCount * 1, timescale: Int32(fps))
let second = CMTimeGetSeconds(frameTime)
print(second)
guard let buffer = pixelBuffer(for: cgImg) else{
continue
}
if !adaptor.append(buffer, withPresentationTime: frameTime) {
print("Failed to append buffer. [image : \(cgImg)]")
}
frameCount += 1
}
videoWriterInput.markAsFinished()
//videoWriter.endSession(atSourceTime: CMTimeMake(value: frameCount * Int64(fps) * durationForEachImage, timescale: fps))
videoWriter.endSession(atSourceTime: CMTimeMake(value: frameCount * 1, timescale: Int32(fps)))
videoWriter.finishWriting {
print("Finish writing")
self.deleteFIles(withExtension: "jpg", inDirectory: documentsPath)
}
}
補足
構造体を使用した理由としては
ドキュメント内に保存したファイルを名前順並び替える為です。
出力した動画ファイルを削除するコードがありますが、既に動画ファイルがある場合とない場合でコメントアウトしたり、しなかったりしないと挙動がおかしくなります。
（修正ポイント）