前回の記事
改良
ただランドマーク打つだけじゃつまんないので改良します。
・いろいろちゃんと関数にした
・顔を目が水平になるように回転
・顔をクリップ・4倍に拡大
・ランドマークと、いろんなもの表示
制約
・二次元しか勝たん。(二次元専用!)
・画像に複数顔がある場合は動作しません。(手抜き)
・お姉さん人物の画像にしか使えません。
準備
作業フォルダに以下のフォルダを作ってください。
・ input : お宝画像をここへ放り込んでください。
・ output : 結果の画像をここへ作ります。
プログラム
import cv2
import glob
import os
import numpy as np
import math
from anime_face_detector import create_detector
from insightface.app.common import Face
from insightface.model_zoo.landmark import Landmark
LEFT_EYE = 38
RIGHT_EYE = 88
NOSE = 86
MOUTH = 60
MOUTH_LEFT = 52
MOUTH_RIGHT = 61
def get_landmarks(landmark_model, image, pred):
bx1, by1, bx2, by2, score = pred["bbox"]
bbox = np.array([bx1, by1, bx2, by2], dtype=np.float32)
face = Face(bbox=bbox, det_score=float(score))
landmark_model.get(image, face)
return face.landmark_2d_106
def expand_image(image, crop_area, margin, scale):
x1, y1, x2, y2 = crop_area
x1 = max(0, x1 - margin)
y1 = max(0, y1 - margin)
x2 = min(image.shape[1], x2 + margin)
y2 = min(image.shape[0], y2 + margin)
face_crop = image[y1:y2, x1:x2]
expanded_image = cv2.resize(
face_crop,
None,
fx=scale, fy=scale,
interpolation=cv2.INTER_NEAREST
)
return expanded_image, [x1, y1]
def expand_landmarks(landmarks, top_left, scale):
expanded_landmarks = landmarks.copy()
for i, point in enumerate(landmarks):
expanded_landmarks[i] = transform_coord(point, top_left, scale)
return expanded_landmarks
def transform_coord(point, top_left, scale):
x = (point[0] - top_left[0]) * scale
y = (point[1] - top_left[1]) * scale
return [x, y]
def plot_landmarks(image, landmarks, top_left, scale):
for j, point in enumerate(landmarks):
x, y = point.astype(int)
cv2.circle(
image,
(x, y),
2,
(0, 0, 255),
-1
)
cv2.putText(
image,
str(j),
(x + 3, y + 12),
cv2.FONT_HERSHEY_SIMPLEX,
0.8,
(0, 127, 0),
2,
cv2.LINE_AA
)
return image
def detect_face_roll_direction(landmarks):
left_eye = np.array(landmarks[LEFT_EYE], dtype=np.float32)
right_eye = np.array(landmarks[RIGHT_EYE], dtype=np.float32)
nose = np.array(landmarks[NOSE], dtype=np.float32)
score = get_projection_parameter(left_eye, right_eye, nose)
if score <= 0.45:
direction = "LEFT"
elif score > 0.55:
direction = "RIGHT"
else:
direction = "FRONT"
return direction, score
def detect_face_pitch_direction(landmarks):
left_eye = np.array(landmarks[LEFT_EYE], dtype=np.float32)
right_eye = np.array(landmarks[RIGHT_EYE], dtype=np.float32)
nose = np.array(landmarks[NOSE], dtype=np.float32)
mouth_left = np.array(landmarks[MOUTH_LEFT], dtype=np.float32)
mouth_right = np.array(landmarks[MOUTH_RIGHT], dtype=np.float32)
eye_center = (left_eye + right_eye) / 2.0
mouth_center = (mouth_left + mouth_right) / 2.0
score = get_projection_parameter(eye_center, mouth_center, nose)
if score <= 0.45:
direction = "UP"
elif score > 0.55:
direction = "DOWN"
else:
direction = "FRONT"
return direction, score
def get_virtical_line(p, q, length):
dx = q[0] - p[0]
dy = q[1] - p[1]
nx = -dy
ny = dx
norm = math.hypot(nx, ny)
nx /= norm
ny /= norm
mx = (p[0] + q[0]) / 2.0
my = (p[1] + q[1]) / 2.0
half = length / 2.0
a = (mx - nx * half, my - ny * half)
b = (mx + nx * half, my + ny * half)
return a, b
def draw_face_direction(image, landmarks, roll, pitch, score_roll, score_pitch):
left_eye = np.array(landmarks[LEFT_EYE], dtype=np.float32)
right_eye = np.array(landmarks[RIGHT_EYE], dtype=np.float32)
nose = np.array(landmarks[NOSE], dtype=np.float32)
mouth_left = np.array(landmarks[MOUTH_LEFT], dtype=np.float32)
mouth_right = np.array(landmarks[MOUTH_RIGHT], dtype=np.float32)
eye_center = (
(left_eye.astype(np.float32) +
right_eye.astype(np.float32)) / 2.0
).astype(np.int32)
mouth_center = (
(mouth_left.astype(np.float32) +
mouth_right.astype(np.float32)) / 2.0
).astype(np.int32)
face_center = (
(eye_center.astype(np.float32) +
mouth_center.astype(np.float32)) / 2.0
).astype(np.int32)
face_center_left = (
face_center.astype(np.float32) - (50.0, 0.0)
).astype(np.int32)
face_center_right = (
face_center.astype(np.float32) + (50.0, 0.0)
).astype(np.int32)
# 左右の目の横線
cv2.line(
image,
tuple(left_eye.astype(np.int32)),
tuple(right_eye.astype(np.int32)),
(0, 255, 0),
3
)
# 左右の口角の横線
cv2.line(
image,
tuple(mouth_left.astype(np.int32)),
tuple(mouth_right.astype(np.int32)),
(0, 255, 0),
3
)
# 顔中心の縦線
cv2.line(
image,
tuple(eye_center.astype(np.int32)),
tuple(mouth_center.astype(np.int32)),
(0, 255, 0),
3
)
# 顔中心の横線
a, b = get_virtical_line(eye_center, mouth_center, 80)
cv2.line(
image,
(round(a[0]), round(a[1])),
(round(b[0]), round(b[1])),
(0, 255, 0),
3
)
# 左右の目の中心
cv2.circle(
image,
tuple(eye_center.astype(np.int32)),
5,
(255, 0, 0),
-1
)
# 鼻
cv2.circle(
image,
tuple(nose.astype(np.int32)),
5,
(255, 0, 0),
-1
)
# 左右の口角の中心
cv2.circle(
image,
tuple(mouth_center),
5,
(255, 0, 0),
-1
)
# ロールの判定とスコア
cv2.putText(
image,
f"ROLL={roll} SCORE={score_roll:.3f}",
(20, 40),
cv2.FONT_HERSHEY_SIMPLEX,
1.0,
(0, 255, 0),
2
)
# ピッチの判定とスコア
cv2.putText(
image,
f"PITCH={pitch} SCORE={score_pitch:.3f}",
(20, 70),
cv2.FONT_HERSHEY_SIMPLEX,
1.0,
(0, 255, 0),
2
)
def get_point(landmarks, number):
return np.array(landmarks[number], dtype=np.float32)
def get_projection_parameter(a, b, p):
d = b - a
t = np.dot(p - a, d) / np.dot(d, d)
return t
def get_angle(p, q):
x1, y1 = p
x2, y2 = q
return math.degrees(math.atan2(y2 - y1, x2 - x1))
def midpoint(p, q):
x1, y1 = p
x2, y2 = q
xm = (x1 + x2) / 2.0
ym = (y1 + y2) / 2.0
return xm, ym
# def rotate_point(point, center, angle):
# x, y = point
# cx, cy = center
# # 回転する中心座標が原点になるように移動
# x -= cx
# y -= cy
# # 回転
# theta = math.radians(angle)
# rx = x * math.cos(theta) - y * math.sin(theta)
# ry = x * math.sin(theta) + y * math.cos(theta)
# # 元の位置に戻す
# rx += cx
# ry += cy
# return rx, ry
def rotate_point(point, center, angle):
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
point_h = np.array([point[0], point[1], 1.0])
return matrix @ point_h
def analysis_face_direction(landmarks):
left_eye = get_point(landmarks, LEFT_EYE)
right_eye = get_point(landmarks, RIGHT_EYE)
nose = get_point(landmarks, NOSE)
mouth_left = get_point(landmarks, MOUTH_LEFT)
mouth_right = get_point(landmarks, MOUTH_RIGHT)
t_eye_nose = get_projection_parameter(left_eye, right_eye, nose)
t_mouth_nose = get_projection_parameter(mouth_left, mouth_right, nose)
eye_center = (left_eye + right_eye) / 2.0
mouth_center = (mouth_left + mouth_right) / 2.0
t_eye_mouth = get_projection_parameter(eye_center, mouth_center, nose)
return t_eye_nose, t_mouth_nose, t_eye_mouth
def get_roll_info(landmarks):
left_eye = get_point(landmarks, LEFT_EYE)
right_eye = get_point(landmarks, RIGHT_EYE)
midpoint_eye = midpoint(left_eye, right_eye)
return midpoint_eye, get_angle(left_eye, right_eye)
def rotate_image(image, center, angle):
h, w = image.shape[:2]
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
rotated = cv2.warpAffine(image, matrix, (w, h))
return rotated
def rotate_landmarks(landmarks, center, angle):
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
points = np.hstack([landmarks, np.ones((len(landmarks), 1))])
correct_landmarks = (matrix @ points.T).T
return correct_landmarks
def main():
detector = create_detector("yolov3", device="cuda:0")
# ユーザ名の部分 xxxxx は適宜修正してください。
landmark_model = Landmark(
r"C:\Users\xxxxx\.insightface\models\buffalo_l\2d106det.onnx"
)
for fn in glob.glob(f"output/*.png"):
os.remove(fn)
for filename in glob.glob("input/*.png"):
b_name = os.path.basename(filename)
o_name = os.path.splitext(b_name)[0]
image = cv2.imread(filename)
# 画像には 1 人しかいないので、preds のループ不要!
# 顔検出 1 回目
preds = detector(image)
if preds is None:
print(f"{b_name}: 顔を検出できませんでした")
continue
pred = preds[0]
landmarks = get_landmarks(landmark_model, image, pred)
# 画像を回転
center, angle = get_roll_info(landmarks)
output = image.copy()
rotated_image = rotate_image(output, center, angle)
# correct_landmarks = rotate_landmarks(landmarks, center, angle)
# 顔検出 2 回目
preds_2 = detector(rotated_image)
if preds_2 is None:
print(f"{b_name}: 顔を検出できませんでした")
continue
pred_2 = preds_2[0]
landmarks_2 = get_landmarks(landmark_model, rotated_image, pred_2)
# クリップ、4 倍に拡大
bx1, by1, bx2, by2, score = pred_2["bbox"]
crop_area = map(int, [bx1, by1, bx2, by2])
margin = 50
scale = 4
expanded_image, top_left = expand_image(rotated_image, crop_area, margin, scale)
expanded_landmarks = expand_landmarks(landmarks_2, top_left, scale)
# ランドマークを描画
output = plot_landmarks(expanded_image, expanded_landmarks, top_left, scale)
# 解析結果を描画
roll, score_roll = detect_face_roll_direction(expanded_landmarks)
pitch, score_pitch = detect_face_pitch_direction(expanded_landmarks)
print(f"{b_name} : Roll={roll} Score={score_roll:.3f} , Pitch={pitch} Score={score_pitch:.3f}")
draw_face_direction(expanded_image, expanded_landmarks, roll, pitch, score_roll, score_pitch)
result = analysis_face_direction(expanded_landmarks)
print(f"{b_name} {result[0]} {result[1]} {result[2]}")
out_filename = f"output/{o_name}_derection.png"
cv2.imwrite(out_filename, output)
# print(f"保存しました: {out_filename}")
if __name__ == "__main__":
main()
判定のしきい値は適当です。
実行結果
大人の事情(意味深)で掲載不可!
BAN されちゃう!
ライセンス
MIT License の下で利用できます。
https://opensource.org/license/mit
※ボクの記事に載せるプログラムはすべて MIT ライセンスです。