PythonにBrain Warsをプレイさせてみた3

はじめに

PythonにBrain Warsの Make10 をプレイさせてみた

Make10 のルール

数字の合計が10になるカードの組み合わせを見つけてタップする

Demo

環境

• OS: Ubuntu 18.04.1 LTS
• Python version: 3.6.5
• スマホ: Huawei P10 Lite

VysorというChromeの拡張プラグインを使って、Ubuntuからスマホを操作している

コード

import mss
import numpy as np
import time
import cv2
import pyautogui as pag
import itertools
from sklearn.neighbors import KNeighborsClassifier


def grab_screen(bbox):
    """Capture the specified area on the screen"""
    with mss.mss() as sct:
        left, top, width, height = bbox
        grab_area = {'left': left, 'top': top, 'width': width, 'height': height}
        img = sct.grab(grab_area)
        return np.array(img)[:, :, :3]


def process_img(img):
    """Extract numbers from each card"""
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    _, img_bin = cv2.threshold(img_gray, 210, 255, cv2.THRESH_BINARY)
    _, cnts, _ = cv2.findContours(img_bin, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    cnts = list(filter(lambda x: 30 < cv2.boundingRect(x)[-1] < 70, cnts))
    cnts = sorted(cnts, key=lambda x: (cv2.boundingRect(x)[1] // 10, cv2.boundingRect(x)[0]))
    rois = []
    locs = []
    for cnt in cnts:
        x, y, w, h = cv2.boundingRect(cnt)
        locs.append([int(x + w/2), int(y + h/2)])
        cv2.rectangle(img, (x, y), (x + w, y + h), (0, 0, 255), 2)
        roi = cv2.resize(img_bin[y:y+h, x:x+w], (10, 10)).reshape(1, 100)
        rois.append(roi)
    cv2.imwrite('detection_result.jpg', img)
    locs = np.array(locs)
    rois = np.array(rois).squeeze()
    return rois, locs


def build_knn():
    """Create and train knn classifier"""
    samples = np.loadtxt('train_samples.data', np.uint8)
    labels = np.loadtxt('train_labels.data', np.float32)
    knn = KNeighborsClassifier(n_neighbors=3)
    knn.fit(samples, labels)
    return knn


def find_combination(nums):
    """Find a combination which sums to 10"""
    result = [seq for i in range(len(nums), 0, -1) for seq in itertools.combinations(nums, i) if sum(seq) == 10]
    return result


def main():
    grab_bbox = (104, 385, 467, 300)
    knn = build_knn()
    while True:    
        img = grab_screen(grab_bbox)
        rois, locs = process_img(img.copy())
        if len(rois) > 12 or len(rois) < 9:
            print('Done')
            break
        nums = knn.predict(rois).astype(int)  # classify numbers
        while len(locs) > 0:
            idx = (np.abs(locs[:, 1] - locs[-1, 1]) < 5)  # index of cards at the bottom row
            locs_ = locs[idx].tolist()
            nums_ = nums[idx].tolist()
            comb = find_combination(nums_)
            print('Cards to click:', comb[0])
            for num in comb[0]:
                idx_click = nums_.index(num)  # index of card to tap
                x, y = (locs_.pop(idx_click))
                nums_.pop(idx_click)
                pag.click(grab_bbox[0] + x, grab_bbox[1] + grab_bbox[3] - 50)  # tap card
            locs = locs[~idx]  # remove cards at the bottom row
            nums = nums[~idx]  
            if len(locs) > 0:
                time.sleep(0.085)
        time.sleep(0.36)


if __name__ == '__main__':
    main()

結果

Python便利