Help us understand the problem. What is going on with this article?

Object co-localization for face recognition

More than 3 years have passed since last update.

In this post, I would like to introduce my implementation of "object co-localization" proposed in
"Co-localization in Real World Images" [Tang+, CVPR2014].
In particular, I used this co-localization for face recognition. The method aims to find specific people who commonly appear in an image set. Object proposals are generated via face detection. Prior is calculated as the probability of skin pixels.

This implementation requires IBM CPlex to solve a binary quadratic programming problem. As suggested in the original paper, you could rely on a continuous QP instead by modifying __disc_clustering function.

colocalize.py
'''
Object co-localization for face recognition
Jan 30, 2015
@jellied_unagi
'''

from pycpx import CPlexModel
import cv2
from skimage.io import ImageCollection
from skimage.feature import local_binary_pattern
from skimage.color import rgb2gray
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import AffinityPropagation
import numpy as np
import matplotlib.pyplot as plt


class colocalize():

    '''
    This class implements "Co-localization in Real World Images" [Tang+, CVPR2014] for localizing people
    who commonly appear in an image set. 
    - Object proposals are generated via face detection.
    - Prior is calculated as the probability of skin pixels.
    - Optimization is done based on IBM CPLex withouth convex relaxation though the original implementation 
    relies on continuous QP instead of BQP.
    '''

    def __init__(self, fsize=100, mn=3, th=0.3, lbp_rad=8, lbp_max=66, method='default', kappa=1e-3, alpha=1e-3):

        # Face detector for object proposals
        self.detector = [cv2.CascadeClassifier(x) 
                         for x in ['haarcascade_frontalface_default.xml',
                                   'haarcascade_frontalface_alt.xml',
                                   'haarcascade_frontalface_alt2.xml', 
                                   'haarcascade_frontalface_alt_tree.xml', 
                                   'haarcascade_profileface.xml']]
        self.fsize = fsize  # minimum face size
        self.mn = mn   # minimum support for detecting faces (larger for more precise detection)
        self.th = th  # threshold for non-maximum supporession

        # Parameters for local binary pattern histograms
        self.lbp_rad = lbp_rad
        self.lbp_np = 8 * lbp_rad
        self.lbp_max = lbp_max
        self.method = method
        self.ss = StandardScaler()

        # Parameters for co-localization
        self.kappa = kappa  # ridge parameter
        self.alpha = alpha  # importance of prior

        # Data holder
        self.image_list = []
        self.face_list = []
        self.feat_list = []
        self.prior_list = []
        self.result_list = []

    def register(self, image):

        '''
        Registering an image to the list
        '''

        print 'Registering image...',
        im_gray = rgb2gray(image)
        face = self.__detect_face(image, size=self.fsize, mn=self.mn, th=self.th)
        if(len(face) == 0):
            print 'Cound not find faces'
            return 0

        skin = self.__detect_skin(image)
        lbp = local_binary_pattern(im_gray, self.lbp_np, self.lbp_rad, self.method)
        feat = []
        prior = []
        for f in face:
            feat.append(np.histogram(lbp[f[1]:f[3], f[0]:f[2]].ravel(), 
                                     self.lbp_max, normed=True)[0])
            prior.append(np.mean(skin[f[1]:f[3], f[0]:f[2]].ravel() / 255.))

        self.image_list.append(image)
        self.face_list.append(face)
        self.feat_list.append(np.vstack(feat))
        self.prior_list.append(np.vstack(prior))

        print 'done.'

    def localize(self):


        '''
        Performing co-localization on the registered images
        '''


        # Scaling features
        self.ss.fit(np.vstack(self.feat_list))
        feat_list = [self.ss.transform(x.copy()) for x in self.feat_list]

        print 'Solving BQP ...',
        idx = self.__disc_clustering(feat_list, self.prior_list)
        self.result_list = [x[y] for (x, y) in zip(self.face_list, idx)]
        print 'done.'

        self.show_results()

    def show_results(self, is_all=True):

        '''
        Visualizing results
        '''

        plt.figure(figsize=(16, 16))
        n_images = len(self.image_list)
        for i in range(n_images):
            if(is_all):
                plt.subplot(np.sqrt(n_images) + 1, np.sqrt(n_images) + 1, i + 1)
            else:
                plt.show()
                plt.figure(figsize=(16, 16))
            img = self.image_list[i]
            face = self.result_list[i]
            plt.imshow(img)
            for f in self.face_list[i]:
                plt.plot([f[0], f[0], f[2], f[2], f[0]],
                         [f[1], f[3], f[3], f[1], f[1]], 'b', lw=6)
            plt.plot([face[0], face[0], face[2], face[2], face[0]],
                     [face[1], face[3], face[3], face[1], face[1]], 'r', lw=6)

            plt.axis('off')

    def __detect_face(self, image, size=80, mn=1, th=0.3):

        '''
        Running the VJ face detector implemented in OpenCV
        '''

        face = [x.detectMultiScale(image, scaleFactor=1.1, minNeighbors=mn,
                                     minSize=(size, size), flags=cv2.cv.CV_HAAR_SCALE_IMAGE) for x in self.detector]
        if (np.sum([len(x) for x in face]) == 0):
            if(size==0):
                print 'Could not find faces'
                return []
            else:
                size = np.max((size - 10, 10))
                mn = np.max((mn - 1, 1))
                print 'searching face...(size %d)' % size
                return self.__detect_face(image, size=size, mn=mn, th=th)

        face = np.vstack([x for x in face if len(x) > 0])
        face[:, 2:] += face[:, :2]
        face = self.__nms(face, th=th)

        return face

    def __nms(self, face, th=.3):

        '''
        non-maximum suppression of detected faces
        '''

        n_faces = len(face)
        fzero = np.zeros((np.max(face[:, 3]), np.max(face[:, 2])))
        fmat = []
        fsum = []
        for i in range(n_faces):
            tmp = fzero.copy()
            tmp[face[i, 1]:face[i, 3], face[i, 0]:face[i, 2]] = 1
            fmat.append(tmp.ravel().astype('bool'))
            fsum.append(np.sum(tmp))

        rem = np.ones(n_faces)
        for i in range(n_faces):
            for j in range(n_faces):
                if i != j:
                    fand = np.sum(fmat[i] & fmat[j])
                    if((fsum[i] < fsum[j]) & ((fand * 1. / fsum[i]) > th)):
                        rem[i] = 0

        return face[rem == 1, :]

    def __disc_clustering(self, feat_list, prior_list):

        '''
        Performing discriminative clustering via BQP
        '''

        X = np.matrix(np.vstack(feat_list))
        nb = X.shape[0] * 1.
        I_nb = np.matrix(np.eye(X.shape[0]))
        I1_nb = np.matrix(np.ones(X.shape[0])).T
        I = np.matrix(np.eye(X.shape[1]))
        cpmat = I_nb - I1_nb * I1_nb.T / nb
        A = cpmat * (I_nb - X * np.linalg.inv(X.T * cpmat * X + nb * self.kappa * I) * X.T) * cpmat / nb
        P = np.matrix(np.vstack(prior_list))
        print np.max(A), np.max(P)

        n_cand = np.array([len(x) for x in feat_list])
        cand_idx = np.hstack((0, np.cumsum(n_cand)))
        B = np.matrix(np.zeros((len(n_cand), A.shape[0])))
        for i in range(len(cand_idx) - 1):
            B[i, cand_idx[i]:cand_idx[i + 1]] = 1

        m = CPlexModel()
        U = m.new((A.shape[0]), vtype = bool)
        b = np.ones(len(n_cand))
        m.constrain(B * U == b)
        m.minimize(U.T * A * U - self.alpha * P.T * U)
        idx = np.argwhere(m[U]==1).flatten() - cand_idx[:-1]

        return idx

    def __detect_skin(self, image):

        '''
        Calculating prior
        '''

        lower = np.array([0, 48, 80], dtype = "uint8")
        upper = np.array([20, 255, 255], dtype = "uint8")
        hsv = cv2.cvtColor(image, cv2.cv.CV_RGB2HSV)
        skinMask = cv2.inRange(hsv, lower, upper)

        return skinMask

Why not register and get more from Qiita?
  1. We will deliver articles that match you
    By following users and tags, you can catch up information on technical fields that you are interested in as a whole
  2. you can read useful information later efficiently
    By "stocking" the articles you like, you can search right away
Comments
Sign up for free and join this conversation.
If you already have a Qiita account
Why do not you register as a user and use Qiita more conveniently?
You need to log in to use this function. Qiita can be used more conveniently after logging in.
You seem to be reading articles frequently this month. Qiita can be used more conveniently after logging in.
  1. We will deliver articles that match you
    By following users and tags, you can catch up information on technical fields that you are interested in as a whole
  2. you can read useful information later efficiently
    By "stocking" the articles you like, you can search right away