javaでopencv使って手の認識

javaで手の認識

javaでopencv叩いて手の平を認識してみた．
ちなみに，描画系がめんどくさいので，processingをライブラリとして使用している．
ソースコードはgithubにあげて置いた．

HsvTest.java

package imageTranslater;

import java.util.ArrayList;

import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfInt;
import org.opencv.core.MatOfInt4;
import org.opencv.core.MatOfPoint;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.highgui.VideoCapture;
import org.opencv.imgproc.Imgproc;

import processing.core.PApplet;

public class HsvTest extends PApplet {

    public static void main(String args[]) {
        PApplet.main(new String[] { "imageTranslater.HsvTest" });
    }

    VideoCapture cap;
    ImageTranslater it;//画像変換ライブラリ

    public void setup() {
        size(1280, 720);
        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);//openCVのロード
        frameRate(60);
        it = new ImageTranslater(this);
        cap = new VideoCapture();
        cap.open(0);
    }

    public void draw() {
        Mat bgr = new Mat();//元画像
        cap.read(bgr);
        Mat hsv = new Mat();//HSV変換画像
        Imgproc.cvtColor(bgr, hsv, Imgproc.COLOR_BGR2HSV);
        Imgproc.medianBlur(hsv, hsv, 3);
        Mat skin = skinDetect(hsv);
        Mat skinDist = distTransform(skin);

        Mat bin = new Mat();
        Imgproc.threshold(skinDist, bin, 0, 255, Imgproc.THRESH_BINARY
                | Imgproc.THRESH_OTSU);

        //肌色の輪郭抽出
        ArrayList<MatOfPoint> contours = new ArrayList<MatOfPoint>();
        Mat hierarchy = new Mat(skinDist.cols(), skinDist.rows(),
                CvType.CV_32SC1);
        Imgproc.findContours(skinDist, contours, hierarchy, Imgproc.RETR_LIST,
                Imgproc.CHAIN_APPROX_NONE);

        //一番大きい肌色の領域を手として認識する
        int index = -1;
        double area = 0;
        for (int i = 0, n = contours.size(); i < n; i++) {
            double tmp = Imgproc.contourArea(contours.get(i));
            if (area < tmp) {
                area = tmp;
                index = i;
            }
        }
        if (index != -1) {
            //手の概形を抽出
            MatOfInt hull = new MatOfInt();
            MatOfPoint contour = contours.get(index);
            Imgproc.convexHull(contour, hull);

            Point data[] = contour.toArray();
            int v = -1;
            for (int i : hull.toArray()) {
                if (v == -1) {
                    v = i;
                } else {
                    Imgproc.line(bgr, data[i], data[v], new Scalar(0, 255, 0));
                    v = i;
                }
            }

            convexityDefects(bgr, contour,hull);
            Imgproc.drawContours(bgr, contours, index, new Scalar(255, 0, 0));
        }
        image(it.Mat2PI(bgr), 0, 0, width, height);
        fill(0);
        text(frameRate + "fps", 0, 10);
    }

    //手の概形を得る
    void convexityDefects(Mat img,MatOfPoint contour,MatOfInt hull) {
        Point data[] = contour.toArray();
        MatOfInt4 convexityDefects = new MatOfInt4();
        Imgproc.convexityDefects(contour, hull, convexityDefects);
        int cd[] = convexityDefects.toArray();
        if(cd==null)return;
        for (int i = 0; i < cd.length; i += 4) {
            Imgproc.line(img, data[cd[i + 1]], data[cd[i + 2]], new Scalar(255, 255, 255));
            Imgproc.line(img, data[cd[i]], data[cd[i + 2]], new Scalar(255, 255, 255));
        }
    }

    //特定閾値内の色を肌色とする
    Mat skinDetect(Mat mat) {
        Mat mat1 = new Mat();
        Core.inRange(mat, new Scalar(0, 58, 89), new Scalar(25, 173, 229), mat1);
        return mat1;
    }

    //距離変換
    Mat distTransform(Mat mat) {
        Mat mat1 = new Mat(mat.cols(), mat.rows(), CvType.CV_8UC1);
        Mat mat2 = new Mat();
        Imgproc.distanceTransform(mat, mat2, Imgproc.CV_DIST_L2, 3);
        Core.convertScaleAbs(mat2, mat1);
        Core.normalize(mat1, mat1, 0.0, 255.0, Core.NORM_MINMAX);
        return mat1;
    }
}

ちなみにopencvの画像オブジェクトとjavaの画像オブジェクト，processingの画像オブジェクトを相互に変換する必要があるので，それらを違うjavaファイルにまとめて書いておく．

ImageTranslater.java

package imageTranslater;

import java.awt.Color;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;

import org.opencv.core.CvType;
import org.opencv.core.Mat;

import processing.core.PApplet;
import processing.core.PImage;

/**
 * javaとprocessingとopencvのための画像オブジェクト変換用クラス
 * ビルド・パスにopencv-xxx.jar（openCV）とcore.jar(Processing)を追加しておいてください．
 *
 * @version 1.1
 */
public class ImageTranslater {
    /**
     * インスタンス化した時にPAppletを保存しておく．
     */
    PApplet pa;

    /**
     * インスタンス化すると短い名前で使用可能です．
     */
    public ImageTranslater(PApplet pa) {
        this.pa = pa;
    }

    /**
     * PImage型をBufferedImage型（TYPE_INT_RGBまたはTYPE_INT_ARGB）に変換します
     *
     * @param img1 変換したいPImage型
     * @return 変換したBufferedImage型
     */
    public static BufferedImage PImageToBufferedImageRGB(PImage img1) {
        int icm = (img1.format == PImage.ARGB) ? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB;
        boolean ifHSB = (img1.format == PImage.HSB);
        BufferedImage img2 = new BufferedImage(img1.width, img1.height, icm);
        img1.loadPixels();
        for (int i = 0; i < img1.width; i++) {
            for (int j = 0; j < img1.height; j++) {
                int color = img1.pixels[i + j * img1.width];
                if (ifHSB) {
                    color = Color.HSBtoRGB((color >> 16) & 0xFF, (color >> 8) & 0xFF, color & 0xFF);
                }
                img2.setRGB(i, j, color);
            }
        }
        return img2;
    }

    /**
     * BufferedImage型をPImage型（RGB）に変換します
     * @param pa PApplet
     * @param img1 変換したいBufferedImage型
     * @return 変換したPImage型
     */
    public static PImage BufferedImageToPImageRGB(PApplet pa, BufferedImage img1) {
        PImage img2 = pa.createImage(img1.getWidth(), img1.getHeight(), pa.RGB);
        for (int i = 0; i < img2.width; i++) {
            for (int j = 0; j < img2.height; j++) {
                img2.pixels[i + j * img2.width] = img1.getRGB(i, j);
            }
        }
        img2.updatePixels();
        return img2;
    }


    /**
     * Mat型をBufferedImage型（TYPE_BYTE_GRAYまたはTYPE_3BYTE_BGR）に変換します
     * @param mat 変換したいMat型
     * @return 変換したBufferedImage
     */
    public static BufferedImage MatToBufferedImageBGR(Mat mat) {
        int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
        byte data[] = new byte[dataSize];
        mat.get(0, 0, data);
        int type;
        if (mat.channels() == 1) {
            type = BufferedImage.TYPE_BYTE_GRAY;
        } else {
            type = BufferedImage.TYPE_3BYTE_BGR;
            for (int i = 0; i < dataSize; i += 3) {
                byte blue = data[i + 0];
                data[i + 0] = data[i + 2];
                data[i + 2] = blue;
            }
        }

        BufferedImage img = new BufferedImage(mat.cols(), mat.rows(), type);
        img.getRaster().setDataElements(0, 0, mat.cols(), mat.rows(), data);
        return img;
    }

    /**
     * Mat型をPImage型（RGB）に変換します
     * @param pa PApplet
     * @param mat 変換したいMat型
     * @return 変換したPImage型
     */
    public static PImage MatToPImageRGB(PApplet pa, Mat mat) {
        PImage img = pa.createImage(mat.cols(), mat.rows(), pa.RGB);
        if (mat.channels() == 1) {
            for(int i=0;i<mat.cols();i++){
                for(int j=0;j<mat.rows();j++){
                    int c=(int)mat.get(j,i)[0];
                    img.pixels[i+j*img.width]=(c<<16)|(c<<8)|c;
                }
            }
        }else{
            for(int i=0;i<mat.cols();i++){
                for(int j=0;j<mat.rows();j++){
                    img.pixels[i+j*img.width]=(int)mat.get(j,i)[0]|((int)mat.get(j,i)[1]<<8)|((int)mat.get(j,i)[2]<<16);
                }
            }
        }
        img.updatePixels();
        return img;
    }

    /**
     * PImage型をMat型（CV_8UC3）に変換します
     * @param img 変換したいPImage型
     * @return 変換したMat型
     */
    public static Mat PImageToMat(PImage img) {
        Mat mat = new Mat(img.height, img.width, CvType.CV_8UC3);
        int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
        byte data[] = new byte[dataSize];
        img.loadPixels();
        int index = 0;
        for (int i = 0; i < dataSize; i += 3) {
            int color = img.pixels[index++];
            data[i + 2] = false ? 0 : (byte) ((color >> 16) & 0xFF);
            data[i + 1] = false ? 0 : (byte) ((color >> 8) & 0xFF);
            data[i] = (byte) (color & 0xFF);
        }
        mat.put(0, 0, data);
        return mat;
    }

    /**
     * PImage型をMat型（CV_8UC3）に変換します
     * @param img 変換したいPImage型
     * @return 変換したMat型
     */
    public static Mat PImageToMatMask(PImage img) {
        Mat mat = new Mat(img.height, img.width, CvType.CV_8U);
        int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
        byte data[] = new byte[dataSize];
        img.loadPixels();
        int index = 0;
        for (int i = 0; i < dataSize; i ++) {
            if((img.pixels[index++]&0x00FFFFFF)!=0);{
                data[i] = 1;
            }
        }
        mat.put(0, 0, data);
        return mat;
    }


    /**
     * BufferedImage型（TYPE_3BYTE_RGB）をMat型（CV_8UC3）に変換します
     * @param image 変換したいBufferedImage型
     * @return 変換したMat型
     */
    public static Mat BufferedImageToMat(BufferedImage image) {
        byte[] data = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
        System.out.println("bufferedimage:" + data.length);
        Mat mat = new Mat(image.getHeight(), image.getWidth(), CvType.CV_8UC3);
        mat.put(0, 0, data);
        return mat;
    }

    public BufferedImage PI2BI(PImage img1) {
        return PImageToBufferedImageRGB(img1);
    }

    public PImage BI2PI(BufferedImage img1) {
        return BufferedImageToPImageRGB(this.pa, img1);
    }

    public BufferedImage Mat2BI(Mat mat) {
        return MatToBufferedImageBGR(mat);
    }

    public PImage Mat2PI(Mat mat) {
        return MatToPImageRGB(this.pa, mat);
    }

    public Mat PI2Mat(PImage img) {
        Mat mat = new Mat(img.height, img.width, CvType.CV_8UC3);
        int dataSize = mat.cols() * mat.rows() * (int) mat.elemSize();
        byte data[] = new byte[dataSize];
        img.loadPixels();
        int index = 0;
        for (int i = 0; i < dataSize; i += 3) {
            int color = img.pixels[index++];
            data[i + 2] = false ? 0 : (byte) ((color >> 16) & 0xFF);
            data[i + 1] = false ? 0 : (byte) ((color >> 8) & 0xFF);
            data[i] = (byte) (color & 0xFF);
        }
        mat.put(0, 0, data);
        return mat;
    }

    public Mat BI2Mat(BufferedImage image) {
        byte[] data = ((DataBufferByte) image.getRaster().getDataBuffer()).getData();
        System.out.println("bufferedimage:" + data.length);
        Mat mat = new Mat(image.getHeight(), image.getWidth(), CvType.CV_8UC3);
        mat.put(0, 0, data);
        return mat;
    }

}