javaでopencv使って手の認識

#javaで手の認識
javaでopencv叩いて手の平を認識してみた．
ちなみに，描画系がめんどくさいので，processingをライブラリとして使用している．
ソースコードはgithubにあげて置いた．

HsvTest.java

package imageTranslater;

import java.util.ArrayList;

import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfInt;
import org.opencv.core.MatOfInt4;
import org.opencv.core.MatOfPoint;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.highgui.VideoCapture;
import org.opencv.imgproc.Imgproc;

import processing.core.PApplet;

public class HsvTest extends PApplet {

	public static void main(String args[]) {
		PApplet.main(new String[] { "imageTranslater.HsvTest" });
	}

	VideoCapture cap;
	ImageTranslater it;//画像変換ライブラリ

	public void setup() {
		size(1280, 720);
		System.loadLibrary(Core.NATIVE_LIBRARY_NAME);//openCVのロード
		frameRate(60);
		it = new ImageTranslater(this);
		cap = new VideoCapture();
		cap.open(0);
	}

	public void draw() {
		Mat bgr = new Mat();//元画像
		cap.read(bgr);
		Mat hsv = new Mat();//HSV変換画像
		Imgproc.cvtColor(bgr, hsv, Imgproc.COLOR_BGR2HSV);
		Imgproc.medianBlur(hsv, hsv, 3);
		Mat skin = skinDetect(hsv);
		Mat skinDist = distTransform(skin);

		Mat bin = new Mat();
		Imgproc.threshold(skinDist, bin, 0, 255, Imgproc.THRESH_BINARY
				| Imgproc.THRESH_OTSU);

		//肌色の輪郭抽出
		ArrayList<MatOfPoint> contours = new ArrayList<MatOfPoint>();
		Mat hierarchy = new Mat(skinDist.cols(), skinDist.rows(),
				CvType.CV_32SC1);
		Imgproc.findContours(skinDist, contours, hierarchy, Imgproc.RETR_LIST,
				Imgproc.CHAIN_APPROX_NONE);

		//一番大きい肌色の領域を手として認識する
		int index = -1;
		double area = 0;
		for (int i = 0, n = contours.size(); i < n; i++) {
			double tmp = Imgproc.contourArea(contours.get(i));
			if (area < tmp) {
				area = tmp;
				index = i;
			}
		}
		if (index != -1) {
			//手の概形を抽出
			MatOfInt hull = new MatOfInt();
			MatOfPoint contour = contours.get(index);
			Imgproc.convexHull(contour, hull);

			Point data[] = contour.toArray();
			int v = -1;
			for (int i : hull.toArray()) {
				if (v == -1) {
					v = i;
				} else {
					Imgproc.line(bgr, data[i], data[v], new Scalar(0, 255, 0));
					v = i;
				}
			}

			convexityDefects(bgr, contour,hull);
			Imgproc.drawContours(bgr, contours, index, new Scalar(255, 0, 0));
		}
		image(it.Mat2PI(bgr), 0, 0, width, height);
		fill(0);
		text(frameRate + "fps", 0, 10);
	}

	//手の概形を得る
	void convexityDefects(Mat img,MatOfPoint contour,MatOfInt hull) {
		Point data[] = contour.toArray();
		MatOfInt4 convexityDefects = new MatOfInt4();
		Imgproc.convexityDefects(contour, hull, convexityDefects);
		int cd[] = convexityDefects.toArray();
		if(cd==null)return;
		for (int i = 0; i < cd.length; i += 4) {
			Imgproc.line(img, data[cd[i + 1]], data[cd[i + 2]], new Scalar(255, 255, 255));
			Imgproc.line(img, data[cd[i]], data[cd[i + 2]], new Scalar(255, 255, 255));
		}
	}

	//特定閾値内の色を肌色とする
	Mat skinDetect(Mat mat) {
		Mat mat1 = new Mat();
		Core.inRange(mat, new Scalar(0, 58, 89), new Scalar(25, 173, 229), mat1);
		return mat1;
	}

	//距離変換
	Mat distTransform(Mat mat) {
		Mat mat1 = new Mat(mat.cols(), mat.rows(), CvType.CV_8UC1);
		Mat mat2 = new Mat();
		Imgproc.distanceTransform(mat, mat2, Imgproc.CV_DIST_L2, 3);
		Core.convertScaleAbs(mat2, mat1);
		Core.normalize(mat1, mat1, 0.0, 255.0, Core.NORM_MINMAX);
		return mat1;
	}
}

ちなみにopencvの画像オブジェクトとjavaの画像オブジェクト，processingの画像オブジェクトを相互に変換する必要があるので，それらを違うjavaファイルにまとめて書いておく．

ImageTranslater.java

package imageTranslater;

import java.awt.Color;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;

import org.opencv.core.CvType;
import org.opencv.core.Mat;

import processing.core.PApplet;
import processing.core.PImage;

/**
 * javaとprocessingとopencvのための画像オブジェクト変換用クラス
 * ビルド・パスにopencv-xxx.jar（openCV）とcore.jar(Processing)を追加しておいてください．
 *
 * @version 1.1
 */
public class ImageTranslater {
	/**
	 * インスタンス化した時にPAppletを保存しておく．
	 */
	PApplet pa;

	/**
	 * インスタンス化すると短い名前で使用可能です．
	 */
	public ImageTranslater(PApplet pa) {
		this.pa = pa;
	}

	/**
	 * PImage型をBufferedImage型（TYPE_INT_RGBまたはTYPE_INT_ARGB）に変換します
	 *
	 * @param img1 変換したいPImage型
	 * @return 変換したBufferedImage型
	 */
	public static BufferedImage PImageToBufferedImageRGB(PImage img1) {
		int icm = (img1.format == PImage.ARGB) ? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB;
		boolean ifHSB = (img1.format == PImage.HSB);
		BufferedImage img2 = new BufferedImage(img1.width, img1.height, icm);
		img1.loadPixels();
		for (int i = 0; i < img1.width; i++) {
			for (int j = 0; j < img1.height; j++) {
				int color = img1.pixels[i + j * img1.width];
				if (ifHSB) {
					color = Color.HSBtoRGB((color >> 16) & 0xFF, (color >> 8) & 0xFF, color & 0xFF);
				}
				img2.setRGB(i, j, color);
			}
		}
		return img2;
	}

	/**
	 * BufferedImage型をPImage型（RGB）に変換します
	 * @param pa PApplet
	 * @param img1 変換したいBufferedImage型
	 * @return 変換したPImage型
	 */
	public static PImage BufferedImageToPImageRGB(PApplet pa, BufferedImage img1) {
		PImage img2 = pa.createImage(img1.getWidth(), img1.getHeight(), pa.RGB);
		for (int i = 0; i < img2.width; i++) {
			for (int j = 0; j < img2.height; j++) {
				img2.pixels[i + j * img2.width] = img1.getRGB(i, j);
			}
		}
		img2.updatePixels();
		return img2;
	}


	/**
	 * Mat型をBufferedImage型（TYPE_BYTE_GRAYまたはTYPE_3BYTE_BGR）に変換します
	 * @param mat 変換したいMat型
	 * @return 変換したBufferedImage
	 */
	public static BufferedImage MatToBufferedImageBGR(Mat mat) {
		int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
		byte data[] = new byte[dataSize];
		mat.get(0, 0, data);
		int type;
		if (mat.channels() == 1) {
			type = BufferedImage.TYPE_BYTE_GRAY;
		} else {
			type = BufferedImage.TYPE_3BYTE_BGR;
			for (int i = 0; i < dataSize; i += 3) {
				byte blue = data[i + 0];
				data[i + 0] = data[i + 2];
				data[i + 2] = blue;
			}
		}

		BufferedImage img = new BufferedImage(mat.cols(), mat.rows(), type);
		img.getRaster().setDataElements(0, 0, mat.cols(), mat.rows(), data);
		return img;
	}

	/**
	 * Mat型をPImage型（RGB）に変換します
	 * @param pa PApplet
	 * @param mat 変換したいMat型
	 * @return 変換したPImage型
	 */
	public static PImage MatToPImageRGB(PApplet pa, Mat mat) {
		PImage img = pa.createImage(mat.cols(), mat.rows(), pa.RGB);
		if (mat.channels() == 1) {
			for(int i=0;i<mat.cols();i++){
				for(int j=0;j<mat.rows();j++){
					int c=(int)mat.get(j,i)[0];
					img.pixels[i+j*img.width]=(c<<16)|(c<<8)|c;
				}
			}
		}else{
			for(int i=0;i<mat.cols();i++){
				for(int j=0;j<mat.rows();j++){
					img.pixels[i+j*img.width]=(int)mat.get(j,i)[0]|((int)mat.get(j,i)[1]<<8)|((int)mat.get(j,i)[2]<<16);
				}
			}
		}
		img.updatePixels();
		return img;
	}

	/**
	 * PImage型をMat型（CV_8UC3）に変換します
	 * @param img 変換したいPImage型
	 * @return 変換したMat型
	 */
	public static Mat PImageToMat(PImage img) {
		Mat mat = new Mat(img.height, img.width, CvType.CV_8UC3);
		int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
		byte data[] = new byte[dataSize];
		img.loadPixels();
		int index = 0;
		for (int i = 0; i < dataSize; i += 3) {
			int color = img.pixels[index++];
			data[i + 2] = false ? 0 : (byte) ((color >> 16) & 0xFF);
			data[i + 1] = false ? 0 : (byte) ((color >> 8) & 0xFF);
			data[i] = (byte) (color & 0xFF);
		}
		mat.put(0, 0, data);
		return mat;
	}

	/**
	 * PImage型をMat型（CV_8UC3）に変換します
	 * @param img 変換したいPImage型
	 * @return 変換したMat型
	 */
	public static Mat PImageToMatMask(PImage img) {
		Mat mat = new Mat(img.height, img.width, CvType.CV_8U);
		int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
		byte data[] = new byte[dataSize];
		img.loadPixels();
		int index = 0;
		for (int i = 0; i < dataSize; i ++) {
			if((img.pixels[index++]&0x00FFFFFF)!=0);{
				data[i] = 1;
			}
		}
		mat.put(0, 0, data);
		return mat;
	}


	/**
	 * BufferedImage型（TYPE_3BYTE_RGB）をMat型（CV_8UC3）に変換します
	 * @param image 変換したいBufferedImage型
	 * @return 変換したMat型
	 */
	public static Mat BufferedImageToMat(BufferedImage image) {
		byte[] data = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
		System.out.println("bufferedimage:" + data.length);
		Mat mat = new Mat(image.getHeight(), image.getWidth(), CvType.CV_8UC3);
		mat.put(0, 0, data);
		return mat;
	}

	public BufferedImage PI2BI(PImage img1) {
		return PImageToBufferedImageRGB(img1);
	}

	public PImage BI2PI(BufferedImage img1) {
		return BufferedImageToPImageRGB(this.pa, img1);
	}

	public BufferedImage Mat2BI(Mat mat) {
		return MatToBufferedImageBGR(mat);
	}

	public PImage Mat2PI(Mat mat) {
		return MatToPImageRGB(this.pa, mat);
	}

	public Mat PI2Mat(PImage img) {
		Mat mat = new Mat(img.height, img.width, CvType.CV_8UC3);
		int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
		byte data[] = new byte[dataSize];
		img.loadPixels();
		int index = 0;
		for (int i = 0; i < dataSize; i += 3) {
			int color = img.pixels[index++];
			data[i + 2] = false ? 0 : (byte) ((color >> 16) & 0xFF);
			data[i + 1] = false ? 0 : (byte) ((color >> 8) & 0xFF);
			data[i] = (byte) (color & 0xFF);
		}
		mat.put(0, 0, data);
		return mat;
	}

	public Mat BI2Mat(BufferedImage image) {
		byte[] data = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
		System.out.println("bufferedimage:" + data.length);
		Mat mat = new Mat(image.getHeight(), image.getWidth(), CvType.CV_8UC3);
		mat.put(0, 0, data);
		return mat;
	}

}