mediapipeで日本手話の指文字を読み取ってみた
自作モデルをmediapipeに追加して手型分類→パターンマッチで指文字を読み取ってみました。あいうえお...わをん、濁音、半濁音、小文字、「ー(長音記号)」で85パターンの認識です。ご参考になれば幸いです。
↓youtubeで動作をご紹介
環境と結果概要
- python：3.8.5
- mediapipe：0.8.2
- mac：Catalina10.15.7、MacBook Pro
- opencv-python：4.4.0
- numpy：1.19.3
- Pillow：8.0.1
参考にさせて頂いたページ・資料
機能概要
- MLPで手型を推定 → landmarksに対するパターンマッチで指文字を推定（右手のみ）
- 手型が固定されたら ↑→↓ の動きを判定、指文字＋動きで入力文字を決定（およそ1秒で1文字）
- 両手の認識が外れたら入力文字を全削除、両手を認識したら入力文字から[−2]の位置を削除（右手で入力、左手で1文字削除で入力文字を調整）
やってみてダメだったこと
- 55パターンでの分類、画像分類の転移学習、MLPともにダメ、今回のモデルは15くらいに手型をグループ分けしてMLPで分類
- 動画撮影用の顔隠しのためholistic（faceとhandとposeを一度に実行）を検証、handの精度が落ちて動作しなくなるため、今回はfaceとhandを個別実行
- faceでの口型分類（「あいうえお」の口型を分類）
未検証、精度や判定速度が上がりそうなもの
- 手型をキュー（履歴）に入れて最頻値での手型判定
- 右手のランドマークを厳密に取得、時々左手として認識されている、ランドマークの番号が狂うので精度低下につながる
- MLPのデータセットを画像表示 → 不適切なデータの削除
- 動作判定を機械学習モデルに変更、左下↙、右上⤴、上↑、右→、後↴が必要
- 手型固定か変化中かの判断モデル作成、変化中は型の判定や動作の判定を除外したい
- 複数人でのデータ取得、取得環境の変更 ※今回テレワーク用の机でデータ取得、こたつで実行すると認識精度が落ちる、妻が実行すると8割くらいの精度
コード
メイン、faceとhandを実行
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import pprint
import csv
import copy
import itertools
from collections import Counter
import cv2 as cv
import numpy as np
import mediapipe as mp
import datetime
from utils import CvFpsCalc
from utils import draw
from utils import hand
from model import KeyPointClassifier
def main():
# 処理用の変数 #
mode = 0
out_no = 87
v = hand.Variables(out_no)
# モデルロード #
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.7, min_tracking_confidence=0.5,)
keypoint_classifier = KeyPointClassifier("model/keypoint_classifier/hand_keypoint_classifier.tflite")
# 動画撮影時の顔隠し用にfaceを使用 #
img_ama = cv.imread("ama.png", cv.IMREAD_UNCHANGED)
mp_face = mp.solutions.face_mesh
face = mp_face.FaceMesh(min_detection_confidence=0.5, min_tracking_confidence=0.5)
# ラベル読み込み #
with open('model/keypoint_classifier/hand_keypoint_classifier_label.csv', encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [row[0] for row in keypoint_classifier_labels]
# カメラ準備 #
cap = cv.VideoCapture(0)
cap.set(cv.CAP_PROP_FRAME_WIDTH, 960)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, 540)
# FPS計測モジュール #
cvFpsCalc = CvFpsCalc(buffer_len=10)
# 処理開始#
while True:
fps = cvFpsCalc.get()
# キー処理(ESC：終了) #
key = cv.waitKey(5)
if key == 27: break #ESC
number, mode = select_mode(key, mode)
# カメラキャプチャ #
ret, image = cap.read()
if not ret: break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
face_results = face.process(image)
image.flags.writeable = True
# 顔を隠す処理 #
if face_results.multi_face_landmarks:
for face_landmarks in face_results.multi_face_landmarks:
face_brect = calc_bounding_rect(image, face_landmarks)
debug_image = draw.overlay_Image(debug_image, img_ama, face_brect)
# 手の処理 #
if results.multi_hand_landmarks is not None and mode != 9:
left_fg, right_fg = 0,0
for hand_landmarks, handedness in zip(results.multi_hand_landmarks, results.multi_handedness):
# 外接矩形の計算、ランドマークの計算、相対座標・正規化座標への変換、学習データ保存 #
brect = calc_bounding_rect(debug_image, hand_landmarks)
landmark_list = calc_landmark_list(debug_image, hand_landmarks)
pre_processed_landmark_list = pre_process_landmark(landmark_list)
logging_csv(number, mode, pre_processed_landmark_list)
# 手型分類、左手は入力文字を1文字削除するフラグとして使用 #
if handedness.classification[0].label[0:] == "Left":
left_fg = 1
v.hand_sign_id = out_no
# 手型分類、右手のみ、ひらがな入力 #
else:
right_fg = 1
#手型分類、分類後のパターンマッチのために上下左右のランドマークポイント取得 #
v.hand_sign_id = keypoint_classifier(pre_processed_landmark_list)
landmark_tips = get_landmark_tips(image,hand_landmarks)
#手型判定 #
v.get_hand_sign_id(landmark_tips,hand_landmarks)
#濁音、半濁音等のために手の動きを判定 #
landmarks_ave = calc_landmarks_ave(hand_landmarks)
v.get_move_id(hand_landmarks, landmarks_ave)
#手型＋動きで判定 #
v.fix_hand_sign_id()
#入力判定 #
if left_fg == 0: v.get_input_letters(keypoint_classifier_labels)
# 手型の判定結果描画 #
brect_center = calc_brect_center(brect)
v.input_letters_position_x = round(brect_center[0])
debug_image = draw.draw_landmarks(debug_image, landmark_list)
debug_image = draw.draw_jp_brect(debug_image, brect, keypoint_classifier_labels[v.hand_sign_id])
# 両手を検出しているときは入力文字を1文字ずつ削除 #
if left_fg == 1 and right_fg == 1:
v.hand_sign_id = out_no -1
if get_tm_int() - v.del_lock_tm > 100 and 2 <= len(v.input_letters):
v.input_letters = v.input_letters[:-2] + v.input_letters[-1]
v.del_lock_tm = get_tm_int()
# 手を認識していないときは入力文字を全削除 #
else:
if get_tm_int() - v.del_lock_tm > 300:
v.input_letters = ""
v.del_lock_tm = get_tm_int()
# fps、入力モード、入力時の番号、手型入力結果の描画
debug_image = draw.draw_info(debug_image, fps, mode, number)
debug_image = draw.draw_jp_input_letters(debug_image, v.input_letters)
# 画面反映 #############################################################
cv.imshow('Hand Gesture Recognition', debug_image)
# 終了処理 #
cap.release()
face.close()
hands.close()
cv.destroyAllWindows()
def get_tm_int():
return int(datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')[:16])
def get_landmark_tips(image, landmarks):
up_num, down_num, left_num, right_num = 0,0,0,0
up_base, down_base, left_base, right_base = image.shape[0], 0 ,image.shape[1], 0
for i,landmark in enumerate(landmarks.landmark):
if up_base >= landmark.y:
up_num, up_base = i, landmark.y
if down_base <= landmark.y:
down_num, down_base = i, landmark.y
if left_base >= landmark.x:
left_num, left_base = i, landmark.x
if right_base <= landmark.x:
right_num, right_base = i, landmark.x
return [up_num, down_num, left_num, right_num]
def select_mode(key, mode):
number = -1
if 48 <= key <= 57: # 0 ~ 9
number = key - 48
if key == 110: # n
mode = 0
if key == 107: # k
mode = 1
if key == ord("s"):
mode = 9
return number, mode
def calc_bounding_rect(image, landmarks):
image_width, image_height = image.shape[1], image.shape[0]
landmark_array = np.empty((0, 2), int)
for _, landmark in enumerate(landmarks.landmark):
landmark_x = min(int(landmark.x * image_width), image_width - 1)
landmark_y = min(int(landmark.y * image_height), image_height - 1)
landmark_point = [np.array((landmark_x, landmark_y))]
landmark_array = np.append(landmark_array, landmark_point, axis=0)
x, y, w, h = cv.boundingRect(landmark_array)
return [x, y, x + w, y + h]
def calc_brect_center(brect):
center_x = (brect[0] + brect[2]) /2
center_y = (brect[1] + brect[3]) /2
return [center_x , center_y]
def calc_landmark_list(image, landmarks):
image_width, image_height = image.shape[1], image.shape[0]
landmark_point = []
for _, landmark in enumerate(landmarks.landmark):
landmark_x = min(int(landmark.x * image_width), image_width - 1)
landmark_y = min(int(landmark.y * image_height), image_height - 1)
landmark_z = landmark.z
landmark_point.append([landmark_x, landmark_y, landmark_z])
return landmark_point
def calc_landmarks_ave(landmarks):
sum_x, sum_y, sum_z = 0,0,0
for _, landmark in enumerate(landmarks.landmark):
sum_x = sum_x + landmark.x
sum_y = sum_y + landmark.y
sum_z = sum_z + landmark.z
ave_x, ave_y, ave_z = sum_x/21, sum_y/21, sum_z/21
return [ave_x, ave_y, ave_z]
def pre_process_landmark(landmark_list):
temp_landmark_list = copy.deepcopy(landmark_list)
# 相対座標に変換
base_x, base_y, base_z = 0, 0, 0
for index, landmark_point in enumerate(temp_landmark_list):
if index == 0:
base_x, base_y, base_z = landmark_point[0], landmark_point[1], landmark_point[2]
temp_landmark_list[index][0] = temp_landmark_list[index][0] - base_x
temp_landmark_list[index][1] = temp_landmark_list[index][1] - base_y
temp_landmark_list[index][2] = (temp_landmark_list[index][2] - base_z)*200
# 1次元リストに変換
temp_landmark_list = list(
itertools.chain.from_iterable(temp_landmark_list))
# 正規化
max_value = max(list(map(abs, temp_landmark_list)))
def normalize_(n): return n / max_value
temp_landmark_list = list(map(normalize_, temp_landmark_list))
return temp_landmark_list
def logging_csv(number, mode, landmark_list):
if mode == 0: pass
if mode == 1 and (0 <= number <= 9):
csv_path = 'model/keypoint_classifier/keypoint.csv'
with open(csv_path, 'a', newline="") as f:
writer = csv.writer(f)
writer.writerow([number, *landmark_list])
if __name__ == '__main__':
main()
指文字判定
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import datetime
from collections import deque
import numpy as np
#### 指文字認識用の変数を取り扱うクラス ####
class Variables:
def __init__(self, out_number):
self.__hand_sign_id = out_number
self.__pre_hand_number = out_number
self.__point_history = deque(maxlen=12)
self.__move_id = 0
self.__move_lock_tm = self.get_tm_int()
self.__input_letters = ""
self.__input_letters_position_x = 0
self.__letter_lock_id = out_number
self.__letter_lock_tm = self.get_tm_int()
self.__del_lock_tm = self.get_tm_int()
@property
def hand_sign_id(self): return self.__hand_sign_id
@hand_sign_id.setter
def hand_sign_id(self, val): self.__hand_sign_id = val
@property
def pre_hand_number(self): return self.__pre_hand_number
@pre_hand_number.setter
def pre_hand_number(self, val): self.__pre_hand_number = val
@property
def point_history(self): return self.__point_history
@point_history.setter
def point_history(self, val): self.__point_history = val
@property
def move_id(self): return self.__move_id
@move_id.setter
def move_id(self, val): self.__move_id = val
@property
def move_lock_tm(self): return self.__move_lock_tm
@move_lock_tm.setter
def move_lock_tm(self, val): self.__move_lock_tm = val
@property
def input_letters(self): return self.__input_letters
@input_letters.setter
def input_letters(self, val): self.__input_letters = val
@property
def input_letters_position_x(self): return self.__input_letters_position_x
@input_letters_position_x.setter
def input_letters_position_x(self, val): self.__input_letters_position_x = val
@property
def letter_lock_id(self): return self.__letter_lock_id
@letter_lock_id.setter
def letter_lock_id(self, val): self.__letter_lock_id = val
@property
def letter_lock_tm(self): return self.__letter_lock_tm
@letter_lock_tm.setter
def letter_lock_tm(self, val): self.__letter_lock_tm = val
@property
def del_lock_tm(self): return self.__del_lock_tm
@del_lock_tm.setter
def del_lock_tm(self, val): self.__del_lock_tm = val
def get_move_id(self, hand_landmarks,landmarks_ave):
if self.pre_hand_number != self.hand_sign_id:
if self.get_tm_int() - self.move_lock_tm > 100:
for i in range(12): self.point_history.append(landmarks_ave)
self.pre_hand_number = self.hand_sign_id
self.move_lock_tm = self.get_tm_int()
self.move_id = 0
elif self.move_id == 0:
t_v1,t_v2 = 1.03, 1.05
self.point_history.append(landmarks_ave)
sum_p1 = np.mean(np.array(list(self.point_history)[0:3]),axis=0)
sum_p2 = np.mean(np.array(list(self.point_history)[4:7]),axis=0)
sum_p3 = np.mean(np.array(list(self.point_history)[8:11]),axis=0)
if sum_p1[0] < sum_p2[0] and sum_p2[0]*t_v1 < sum_p3[0] and sum_p1[0]*t_v2 < sum_p3[0]: self.move_id = 2 #濁音
elif sum_p1[1] < sum_p2[1] and sum_p2[1]*t_v1 < sum_p3[1] and sum_p1[1]*t_v2 < sum_p3[1]: self.move_id = 1 #を、小文字
elif sum_p3[1] < sum_p2[1] and sum_p2[1]*t_v1 < sum_p1[1] and sum_p3[1]*t_v2 < sum_p1[1]: self.move_id = 3 #半濁音
print("move_id : " + str(self.move_id))
self.move_unlock_tm = self.get_tm_int()
elif self.get_tm_int() - self.move_unlock_tm > 100:
self.move_id = 0
for i in range(12): self.point_history.append(landmarks_ave)
def get_hand_sign_id(self, landmark_tips, hand_landmarks):
if self.hand_sign_id == 0:#いきちつめ
if landmark_tips[0] in [8]: self.hand_sign_id = 16 #き
elif hand_landmarks.landmark[4].y <= hand_landmarks.landmark[15].y: self.hand_sign_id = 11 #い
elif hand_landmarks.landmark[12].y <= hand_landmarks.landmark[19].y: self.hand_sign_id = 43 #め
elif hand_landmarks.landmark[16].y <= hand_landmarks.landmark[19].y: self.hand_sign_id = 27 #つ
else: self.hand_sign_id = 26 #ち
elif self.hand_sign_id == 1:#かそのぬひん
if landmark_tips[2] in [8]: self.hand_sign_id = 34 #の
elif landmark_tips[0] in [5,9]: self.hand_sign_id = 24 #そ
elif landmark_tips[0] in [6,7]: self.hand_sign_id = 32 #ぬ
elif landmark_tips[0] in [8]: #かひん
if hand_landmarks.landmark[12].x <= hand_landmarks.landmark[4].x and not hand_landmarks.landmark[2].y <= hand_landmarks.landmark[12].y: self.hand_sign_id = 15 #か
elif hand_landmarks.landmark[8].x <= hand_landmarks.landmark[10].x: self.hand_sign_id = 36 #ひ
else: self.hand_sign_id = 55 #ん
elif self.hand_sign_id == 2:#うとなにはらり
if landmark_tips[3] in [1,2,5]: self.hand_sign_id = 29 #と
elif landmark_tips[3] in [8,12]: self.hand_sign_id = 35 #は
elif landmark_tips[0] in [12]: #
if hand_landmarks.landmark[8].x <= hand_landmarks.landmark[12].x: self.hand_sign_id = 12 #う
else: self.hand_sign_id = 48 #ら
elif landmark_tips[0] in [0,1,9]: self.hand_sign_id = 30 #な
elif landmark_tips[0] in [5]: self.hand_sign_id = 49 #り
elif landmark_tips[2] in [12]: self.hand_sign_id = 31 #に
elif self.hand_sign_id == 3:#まみゆわ
if landmark_tips[0] in [0,1]: self.hand_sign_id = 40 #ま
elif landmark_tips[2] in [12]: self.hand_sign_id = 41 #み
elif landmark_tips[3] in [1,2,5,6,7]: self.hand_sign_id = 46 #ゆ
elif landmark_tips[2] in [2,8]: self.hand_sign_id = 53 #わ
elif self.hand_sign_id == 5: #あた
if landmark_tips[2] in [4]: self.hand_sign_id = 10 #あ
elif landmark_tips[0] in [4]: self.hand_sign_id = 25 #た
elif self.hand_sign_id == 6: #ふむれ
if landmark_tips[1] in [8]: self.hand_sign_id = 37 #ふ
elif landmark_tips[2] in [8]: self.hand_sign_id = 42 #む
elif landmark_tips[0] in [8]: self.hand_sign_id = 51 #れ
elif self.hand_sign_id == 7: #しする
if landmark_tips[2] in [12]: self.hand_sign_id = 21 #し
elif landmark_tips[1] in [12,17]: self.hand_sign_id = 22 #す
elif landmark_tips[0] in [8,12]: self.hand_sign_id = 50 #る
elif self.hand_sign_id == 8: #へや
if landmark_tips[0] in [0,4,5,9,13]: self.hand_sign_id = 38 #へ
elif landmark_tips[1] in [0]: self.hand_sign_id = 45 #や
elif self.hand_sign_id == 9: #えけてねほよ
if landmark_tips[0] in [4]: self.hand_sign_id = 17 #く
elif landmark_tips[2] in [12]: self.hand_sign_id = 47 #よ
elif landmark_tips[0] in [10,11]: self.hand_sign_id = 13 #え
elif landmark_tips[2] in [17,18,19,20]: self.hand_sign_id = 39 #ほ
elif landmark_tips[2] not in [3,4]: self.hand_sign_id = 18 #け
elif landmark_tips[0] in [12]: self.hand_sign_id = 28 #て
elif landmark_tips[0] in [0]: self.hand_sign_id = 33 #の
def fix_hand_sign_id(self):
if self.move_id != 0:
if self.hand_sign_id == 14 and self.move_id == 1: self.hand_sign_id = 54 #を
elif self.hand_sign_id == 15 and self.move_id == 2: self.hand_sign_id = 56 #が
elif self.hand_sign_id == 16 and self.move_id == 2: self.hand_sign_id = 57 #ぎ
elif self.hand_sign_id == 17 and self.move_id == 2: self.hand_sign_id = 58 #ぐ
elif self.hand_sign_id == 18 and self.move_id == 2: self.hand_sign_id = 59 #げ
elif self.hand_sign_id == 19 and self.move_id == 2: self.hand_sign_id = 60 #ご
elif self.hand_sign_id == 20 and self.move_id == 2: self.hand_sign_id = 61 #ざ
elif self.hand_sign_id == 21 and self.move_id == 2: self.hand_sign_id = 62 #じ
elif self.hand_sign_id == 22 and self.move_id == 2: self.hand_sign_id = 63 #ず
elif self.hand_sign_id == 23 and self.move_id == 2: self.hand_sign_id = 64 #ぜ
elif self.hand_sign_id == 24 and self.move_id == 2: self.hand_sign_id = 65 #ぞ
elif self.hand_sign_id == 25 and self.move_id == 2: self.hand_sign_id = 66 #だ
elif self.hand_sign_id == 26 and self.move_id == 2: self.hand_sign_id = 67 #ぢ
elif self.hand_sign_id == 27 and self.move_id == 2: self.hand_sign_id = 68 #づ
elif self.hand_sign_id == 28 and self.move_id == 2: self.hand_sign_id = 69 #で
elif self.hand_sign_id == 29 and self.move_id == 2: self.hand_sign_id = 70 #ど
elif self.hand_sign_id == 35 and self.move_id == 2: self.hand_sign_id = 71 #ば
elif self.hand_sign_id == 36 and self.move_id == 2: self.hand_sign_id = 72 #び
elif self.hand_sign_id == 37 and self.move_id == 2: self.hand_sign_id = 73 #ぶ
elif self.hand_sign_id == 38 and self.move_id == 2: self.hand_sign_id = 74 #べ
elif self.hand_sign_id == 39 and self.move_id == 2: self.hand_sign_id = 75 #ぼ
elif self.hand_sign_id == 35 and self.move_id == 3: self.hand_sign_id = 76 #ぱ
elif self.hand_sign_id == 36 and self.move_id == 3: self.hand_sign_id = 77 #ぴ
elif self.hand_sign_id == 37 and self.move_id == 3: self.hand_sign_id = 78 #ぷ
elif self.hand_sign_id == 38 and self.move_id == 3: self.hand_sign_id = 79 #ぺ
elif self.hand_sign_id == 39 and self.move_id == 3: self.hand_sign_id = 80 #ぽ
elif self.hand_sign_id == 27 and self.move_id == 1: self.hand_sign_id = 81 #小つ
elif self.hand_sign_id == 45 and self.move_id == 1: self.hand_sign_id = 82 #小や
elif self.hand_sign_id == 46 and self.move_id == 1: self.hand_sign_id = 83 #小ゆ
elif self.hand_sign_id == 47 and self.move_id == 1: self.hand_sign_id = 84 #小よ
elif self.hand_sign_id == 24 and self.move_id == 1: self.hand_sign_id = 85 #ー
def get_input_letters(self, keypoint_classifier_labels):
if self.letter_lock_id == self.hand_sign_id:
if self.get_tm_int() - self.letter_lock_tm > 400:
self.input_letters = self.input_letters + keypoint_classifier_labels[self.hand_sign_id]
self.letter_lock_tm = self.get_tm_int()
elif self.get_tm_int() - self.letter_lock_tm > 20:
if len(self.input_letters) == 0:
self.input_letters = self.input_letters + keypoint_classifier_labels[self.hand_sign_id]
elif self.input_letters[-1] == "お" and self.hand_sign_id == 54: self.input_letters = self.input_letters[0:-1] + "を"
elif self.input_letters[-1] == "か" and self.hand_sign_id == 56: self.input_letters = self.input_letters[0:-1] + "が"
elif self.input_letters[-1] == "き" and self.hand_sign_id == 57: self.input_letters = self.input_letters[0:-1] + "ぎ"
elif self.input_letters[-1] == "く" and self.hand_sign_id == 58: self.input_letters = self.input_letters[0:-1] + "ぐ"
elif self.input_letters[-1] == "け" and self.hand_sign_id == 59: self.input_letters = self.input_letters[0:-1] + "げ"
elif self.input_letters[-1] == "こ" and self.hand_sign_id == 60: self.input_letters = self.input_letters[0:-1] + "ご"
elif self.input_letters[-1] == "さ" and self.hand_sign_id == 61: self.input_letters = self.input_letters[0:-1] + "ざ"
elif self.input_letters[-1] == "し" and self.hand_sign_id == 62: self.input_letters = self.input_letters[0:-1] + "じ"
elif self.input_letters[-1] == "す" and self.hand_sign_id == 63: self.input_letters = self.input_letters[0:-1] + "ず"
elif self.input_letters[-1] == "せ" and self.hand_sign_id == 64: self.input_letters = self.input_letters[0:-1] + "ぜ"
elif self.input_letters[-1] == "そ" and self.hand_sign_id == 65: self.input_letters = self.input_letters[0:-1] + "ぞ"
elif self.input_letters[-1] == "た" and self.hand_sign_id == 66: self.input_letters = self.input_letters[0:-1] + "だ"
elif self.input_letters[-1] == "ち" and self.hand_sign_id == 67: self.input_letters = self.input_letters[0:-1] + "ぢ"
elif self.input_letters[-1] == "つ" and self.hand_sign_id == 68: self.input_letters = self.input_letters[0:-1] + "づ"
elif self.input_letters[-1] == "て" and self.hand_sign_id == 69: self.input_letters = self.input_letters[0:-1] + "で"
elif self.input_letters[-1] == "と" and self.hand_sign_id == 70: self.input_letters = self.input_letters[0:-1] + "ど"
elif self.input_letters[-1] == "は" and self.hand_sign_id == 71: self.input_letters = self.input_letters[0:-1] + "ば"
elif self.input_letters[-1] == "ひ" and self.hand_sign_id == 72: self.input_letters = self.input_letters[0:-1] + "び"
elif self.input_letters[-1] == "ふ" and self.hand_sign_id == 73: self.input_letters = self.input_letters[0:-1] + "ぶ"
elif self.input_letters[-1] == "へ" and self.hand_sign_id == 74: self.input_letters = self.input_letters[0:-1] + "べ"
elif self.input_letters[-1] == "ほ" and self.hand_sign_id == 75: self.input_letters = self.input_letters[0:-1] + "ぼ"
elif self.input_letters[-1] == "は" and self.hand_sign_id == 76: self.input_letters = self.input_letters[0:-1] + "ぱ"
elif self.input_letters[-1] == "ひ" and self.hand_sign_id == 77: self.input_letters = self.input_letters[0:-1] + "ぴ"
elif self.input_letters[-1] == "ふ" and self.hand_sign_id == 78: self.input_letters = self.input_letters[0:-1] + "ぷ"
elif self.input_letters[-1] == "へ" and self.hand_sign_id == 79: self.input_letters = self.input_letters[0:-1] + "ぺ"
elif self.input_letters[-1] == "ほ" and self.hand_sign_id == 80: self.input_letters = self.input_letters[0:-1] + "ぽ"
elif self.input_letters[-1] == "つ" and self.hand_sign_id == 81: self.input_letters = self.input_letters[0:-1] + "っ"
elif self.input_letters[-1] == "や" and self.hand_sign_id == 82: self.input_letters = self.input_letters[0:-1] + "ゃ"
elif self.input_letters[-1] == "ゆ" and self.hand_sign_id == 83: self.input_letters = self.input_letters[0:-1] + "ゅ"
elif self.input_letters[-1] == "よ" and self.hand_sign_id == 84: self.input_letters = self.input_letters[0:-1] + "ょ"
elif self.input_letters[-1] == "そ" and self.hand_sign_id == 85: self.input_letters = self.input_letters[0:-1] + "ー"
elif self.input_letters[-1] != keypoint_classifier_labels[self.hand_sign_id][-1]:
self.input_letters = self.input_letters + keypoint_classifier_labels[self.hand_sign_id]
else:
self.letter_lock_id = self.hand_sign_id
self.letter_lock_tm = self.get_tm_int()
def get_tm_int(self):
return int(datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')[:16])
描画用
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import cv2 as cv
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from math import floor
Font = '/System/Library/Fonts/ヒラギノ丸ゴ ProN W4.ttc'
def get_point_x_y(landmark_point):
return landmark_point[0],landmark_point[1]
def draw_landmarks(image, landmark_point):
# 接続線
if len(landmark_point) > 0:
# 親指
cv.line(image, get_point_x_y(landmark_point[2]), get_point_x_y(landmark_point[3]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[2]), get_point_x_y(landmark_point[3]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[3]), get_point_x_y(landmark_point[4]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[3]), get_point_x_y(landmark_point[4]),
(255, 255, 255), 2)
# 人差指
cv.line(image, get_point_x_y(landmark_point[5]), get_point_x_y(landmark_point[6]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[5]), get_point_x_y(landmark_point[6]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[6]), get_point_x_y(landmark_point[7]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[6]), get_point_x_y(landmark_point[7]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[7]), get_point_x_y(landmark_point[8]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[7]), get_point_x_y(landmark_point[8]),
(255, 255, 255), 2)
# 中指
cv.line(image, get_point_x_y(landmark_point[9]), get_point_x_y(landmark_point[10]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[9]), get_point_x_y(landmark_point[10]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[10]), get_point_x_y(landmark_point[11]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[10]), get_point_x_y(landmark_point[11]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[11]), get_point_x_y(landmark_point[12]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[11]), get_point_x_y(landmark_point[12]),
(255, 255, 255), 2)
# 薬指
cv.line(image, get_point_x_y(landmark_point[13]), get_point_x_y(landmark_point[14]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[13]), get_point_x_y(landmark_point[14]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[14]), get_point_x_y(landmark_point[15]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[14]), get_point_x_y(landmark_point[15]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[15]), get_point_x_y(landmark_point[16]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[15]), get_point_x_y(landmark_point[16]),
(255, 255, 255), 2)
# 小指
cv.line(image, get_point_x_y(landmark_point[17]), get_point_x_y(landmark_point[18]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[17]), get_point_x_y(landmark_point[18]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[18]), get_point_x_y(landmark_point[19]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[18]), get_point_x_y(landmark_point[19]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[19]), get_point_x_y(landmark_point[20]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[19]), get_point_x_y(landmark_point[20]),
(255, 255, 255), 2)
# 手の平
cv.line(image, get_point_x_y(landmark_point[0]), get_point_x_y(landmark_point[1]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[0]), get_point_x_y(landmark_point[1]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[1]), get_point_x_y(landmark_point[2]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[1]), get_point_x_y(landmark_point[2]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[2]), get_point_x_y(landmark_point[5]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[2]), get_point_x_y(landmark_point[5]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[5]), get_point_x_y(landmark_point[9]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[5]), get_point_x_y(landmark_point[9]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[9]), get_point_x_y(landmark_point[13]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[9]), get_point_x_y(landmark_point[13]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[13]), get_point_x_y(landmark_point[17]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[13]), get_point_x_y(landmark_point[17]),
(255, 255, 255), 2)
cv.line(image, get_point_x_y(landmark_point[17]), get_point_x_y(landmark_point[0]),
(0, 0, 0), 6)
cv.line(image, get_point_x_y(landmark_point[17]), get_point_x_y(landmark_point[0]),
(255, 255, 255), 2)
# キーポイント
for index, landmark in enumerate(landmark_point):
if index == 0: # 手首1
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 1: # 手首2
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 2: # 親指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 3: # 親指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 4: # 親指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 5: # 人差指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 6: # 人差指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 7: # 人差指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 8: # 人差指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 9: # 中指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 10: # 中指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 11: # 中指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 12: # 中指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 13: # 薬指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 14: # 薬指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 15: # 薬指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 16: # 薬指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 17: # 小指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 18: # 小指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 19: # 小指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 20: # 小指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
return image
def draw_bounding_rect(image, brect):
cv.rectangle(image, (brect[0], brect[1]), (brect[2], brect[3]),(0, 0, 0), 1)
return image
def draw_info_brect(image, brect):
cv.rectangle(image, (brect[0], brect[1]), (brect[2], brect[1] - 22), (0, 0, 0), -1)
return image
def draw_jp_input_letters(image, str):
if str != "":
margin = 3
font_size = 30
image_width, image_height = image.shape[1], image.shape[0]
#cv.rectangle(
# image,
# (floor(image_width/2 -len(str)*font_size/2+margin-4), floor(image_height/10 - margin)),
# (floor(image_width/2 +len(str)*font_size/2+margin-2), floor(image_height/10 +font_size +margin)),
# (255,255,255), -1)
tmp_Image = Image.fromarray(image)
d = ImageDraw.Draw(tmp_Image)
d.font = ImageFont.truetype(Font, 30)
bw = 1
pos = [image_width / 2 - len(str) * 15, image_height / 10]
color_f = (255,255,255)
color_b = (0,0,0)
d.text((pos[0]-bw,pos[1]-bw),str,color_b)
d.text((pos[0]-bw,pos[1]+bw),str,color_b)
d.text((pos[0]+bw,pos[1]-bw),str,color_b)
d.text((pos[0]+bw,pos[1]+bw),str,color_b)
d.text((pos[0],pos[1]),str,color_f)
image = np.array(tmp_Image)
return image
def draw_jp_brect(image, brect, hand_sign_text):
if hand_sign_text != "":
info_text = hand_sign_text
tmp_Image = Image.fromarray(image)
d = ImageDraw.Draw(tmp_Image)
d.font = ImageFont.truetype(Font, 30)
bw = 1
pos = [round((brect[0] + brect[2])/2), brect[1] - 50]
color_f = (255,255,255)
color_b = (0,0,0)
d.text((pos[0]-bw,pos[1]-bw),info_text,color_b)
d.text((pos[0]-bw,pos[1]+bw),info_text,color_b)
d.text((pos[0]+bw,pos[1]-bw),info_text,color_b)
d.text((pos[0]+bw,pos[1]+bw),info_text,color_b)
d.text((pos[0],pos[1]),info_text,color_f)
image = np.array(tmp_Image)
return image
def draw_info(image, fps, mode, number):
cv.putText(image, "FPS:" + str(fps), (10, 30), cv.FONT_HERSHEY_SIMPLEX,
1.0, (0, 0, 0), 4, cv.LINE_AA)
cv.putText(image, "FPS:" + str(fps), (10, 30), cv.FONT_HERSHEY_SIMPLEX,
1.0, (255, 255, 255), 2, cv.LINE_AA)
if 0 <= mode <= 9:
cv.putText(image, "MODE:" + str(mode), (10, 90),
cv.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1,
cv.LINE_AA)
if 0 <= number <= 9:
cv.putText(image, "NUM:" + str(number), (10, 110),
cv.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1,
cv.LINE_AA)
return image
def overlay_Image(image, overlay, brect):
bw = 10
overlay = cv.resize(overlay, (brect[2] - brect[0] - bw*2, brect[3] - brect[1] - bw))
# 背景をPIL形式に変換
pil_src = Image.fromarray(image)
pil_src = pil_src.convert('RGBA')
# オーバーレイをPIL形式に変換
pil_overlay = Image.fromarray(overlay)
pil_overlay = pil_overlay.convert('RGBA')
# 画像を合成
pil_tmp = Image.new('RGBA', pil_src.size, (0,0,0, 0))
pil_tmp.paste(pil_overlay, (brect[0]+bw,brect[1]-bw*2), pil_overlay)
result_image = Image.alpha_composite(pil_src, pil_tmp)
# OpenCV形式に変換
image = np.array(result_image)
return image
※CSVやモデルのご参考として
- https://github.com/kirei7jaiann/yubimoji/tree/master