はじめに
最近、ポケモンチャンピオンズでポケモンバトルをしています。
対戦前の選出画面から相手の6体を読み取り、自分が選出するポケモンや戦略を考える仕組みを作りたいと思い、開発を進めています。
これまでに、以下の2つを実装しました。
今回は、この2つをWebSocketで接続し、相手6体の名前をターミナルに表示するまでを実装します。
成果物
選出画面を表示してプログラムを実行すると、相手の6体を自動で判定できます。
実際の動作動画
取得したポケモン選出画面
画面右側に相手の6体が並んでいます。この領域を解析します。
ターミナルへの出力結果
認識したポケモンの名前・スコア・判定状態を表示します。
システム構成
今回のシステム関連図です。
SwiftでiPhoneミラーリングの画面を取得し、WebSocketでAPIサーバへ画像を送信します。サーバではOpenCVを使って相手6体を特定し、認識結果をJSONで返却します。
今回はブラウザ表示やLLMによる選出判断は扱いません。
処理フロー
① iPhoneミラーリングの画像を取得
MacのiPhoneミラーリングでポケモンチャンピオンズを操作します。
SwiftのScreenCaptureKitを利用し、ミラーリングのウインドウをPNG画像として取得します。画面に出ている iPhoneミラーリングの窓のうち、面積が最大のものを撮ります。
画面収録の許可や窓の選び方は、過去記事にもまとめています。
iPhoneミラーリングでMacに繋ぎ、ミラーリング時の画像を取得する。
pokemon-champions-swift/Sources/CaptureCore/IPhoneMirroringCapture.swift
import AppKit
import CoreGraphics
import Foundation
import ImageIO
import ScreenCaptureKit
import UniformTypeIdentifiers
struct MirrorWindowCandidate: Equatable {
var windowID: UInt32
var bundleIdentifier: String?
var applicationName: String
var width: CGFloat
var height: CGFloat
var onScreen: Bool
var area: CGFloat { width * height }
}
enum MirrorCaptureError: LocalizedError {
case notOpen
case permission
case unavailable
case encodeFailed
var errorDescription: String? {
switch self {
case .notOpen:
return "iPhoneミラーリングが開いていません"
case .permission:
return "画面収録が許可されていません"
case .unavailable:
return "このMacではiPhoneミラーリングの画面を取得できません"
case .encodeFailed:
return "iPhoneミラーリングの画像をPNGにできませんでした"
}
}
}
func chooseMirrorWindow(_ windows: [MirrorWindowCandidate]) -> MirrorWindowCandidate? {
let mirrors = windows.filter(isIPhoneMirroringWindow).filter { $0.area > 0 }
let visible = mirrors.filter(\.onScreen)
return (visible.isEmpty ? mirrors : visible).max { $0.area < $1.area }
}
func isIPhoneMirroringWindow(_ window: MirrorWindowCandidate) -> Bool {
if window.bundleIdentifier?.caseInsensitiveCompare(iPhoneMirroringBundleID) == .orderedSame {
return true
}
let name = window.applicationName.precomposedStringWithCanonicalMapping
return name.caseInsensitiveCompare("iPhone Mirroring") == .orderedSame
|| name == "iPhoneミラーリング"
}
private let iPhoneMirroringBundleID = "com.apple.ScreenContinuity"
@MainActor
enum IPhoneMirroringCapture {
static func takePNG() async throws -> Data {
_ = NSApplication.shared
if !CGPreflightScreenCaptureAccess() {
_ = CGRequestScreenCaptureAccess()
}
guard CGPreflightScreenCaptureAccess() else {
throw MirrorCaptureError.permission
}
guard #available(macOS 14.0, *) else {
throw MirrorCaptureError.unavailable
}
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
let candidates = content.windows.map(candidate)
guard let chosen = chooseMirrorWindow(candidates),
let window = content.windows.first(where: { $0.windowID == chosen.windowID })
else {
throw MirrorCaptureError.notOpen
}
let filter = SCContentFilter(desktopIndependentWindow: window)
let configuration = SCStreamConfiguration()
configuration.showsCursor = false
let info = SCShareableContent.info(for: filter)
let scale = max(1, CGFloat(info.pointPixelScale))
configuration.width = max(1, Int((info.contentRect.width * scale).rounded()))
configuration.height = max(1, Int((info.contentRect.height * scale).rounded()))
let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
return try pngData(from: image)
}
}
private func candidate(_ window: SCWindow) -> MirrorWindowCandidate {
MirrorWindowCandidate(
windowID: window.windowID,
bundleIdentifier: window.owningApplication?.bundleIdentifier,
applicationName: window.owningApplication?.applicationName ?? "",
width: window.frame.width,
height: window.frame.height,
onScreen: window.isOnScreen
)
}
private func pngData(from image: CGImage) throws -> Data {
let data = NSMutableData()
guard let destination = CGImageDestinationCreateWithData(data, UTType.png.identifier as CFString, 1, nil) else {
throw MirrorCaptureError.encodeFailed
}
CGImageDestinationAddImage(destination, image, nil)
guard CGImageDestinationFinalize(destination) else {
throw MirrorCaptureError.encodeFailed
}
return data as Data
}
② WebSocketで画像を送信
取得したPNG画像を、WebSocketでサーバに送信します。
送信先は以下です。
ws://127.0.0.1:3000/capture
画像はバイナリデータとして送信します。ファイル名も Content-Type も付けません。今回は画像を1枚送信し、その認識結果をテキスト1通で受け取ります。接続が切れたときは、一度だけつなぎ直して同じPNGを送り直します。
入口は send-images です。撮影したPNGを送り、返ったJSONを標準出力へ出します。
pokemon-champions-swift/Sources/SendImages/main.swift
import CaptureCore
@main
enum SendImagesMain {
static func main() async {
await CaptureCore.SendImages.main()
}
}
pokemon-champions-swift/Sources/CaptureCore/ImageBatch.swift
import Foundation
public enum SendImages {
public static func main() async {
let urlString = ProcessInfo.processInfo.environment["CAPTURE_URL"] ?? "ws://127.0.0.1:3000/capture"
guard let url = URL(string: urlString) else {
emit("URL が不正です: \(urlString)")
Foundation.exit(1)
}
do {
let png = try await IPhoneMirroringCapture.takePNG()
try await send(png: png, url: url)
} catch {
emit(error.localizedDescription)
Foundation.exit(1)
}
}
static func send(png: Data, url: URL) async throws {
saveDebugPNG(png)
let client = CaptureClient(url: url, logger: StandardCaptureLogger(), responseTimeout: 180, openTimeout: 5)
try await client.connect()
emit("\(url.absoluteString) へ iPhoneミラーリングを送ります")
emit("iPhoneミラーリング (\(png.count) bytes)")
var trip = await client.roundTrip(png)
if case .disconnected = trip {
try await client.connect()
trip = await client.roundTrip(png)
}
switch trip {
case .text(let text):
emit(describeImageResult(parseServerMessage(text)))
case .timeout:
emit(" 応答がタイムアウトしました")
case .disconnected:
emit(" 接続が切れました")
}
await client.close()
}
/// 送る PNG を、起動したディレクトリの `debug/iphone-mirroring/` に残す。保存の失敗では送るのを止めない。
static func saveDebugPNG(_ png: Data, directory: URL? = nil, now: Date = Date()) {
let folder = directory ?? URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
.appendingPathComponent("debug/iphone-mirroring", isDirectory: true)
do {
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
let file = folder.appendingPathComponent(debugFileName(now: now))
try png.write(to: file, options: .atomic)
emit("保存しました \(file.path)")
} catch {
emit("画像を保存できませんでした: \(error.localizedDescription)")
}
}
static func debugFileName(now: Date) -> String {
let formatter = DateFormatter()
formatter.locale = Locale(identifier: "en_US_POSIX")
formatter.dateFormat = "yyyyMMdd-HHmmss-SSS"
return "\(formatter.string(from: now)).png"
}
}
private func emit(_ line: String) {
FileHandle.standardOutput.write(Data((line + "\n").utf8))
}
遅れて届いた応答は、標準エラーへ残します。
pokemon-champions-swift/Sources/CaptureCore/CaptureLogging.swift
import Foundation
protocol CaptureLogging: Sendable {
func log(_ message: String)
}
struct StandardCaptureLogger: CaptureLogging {
private static let lock = NSLock()
func log(_ message: String) {
Self.lock.lock()
defer { Self.lock.unlock() }
fputs("[capture] \(message)\n", stderr)
fflush(stderr)
}
}
CaptureClient は FrameSending に合わせています。プロトコルは次のとおりです。
pokemon-champions-swift/Sources/CaptureCore/CaptureLoop.swift
protocol FrameSending: Sendable {
func connect() async throws
func roundTrip(_ png: Data) async -> RoundTrip
var isOpen: Bool { get async }
func close() async
}
pokemon-champions-swift/Sources/CaptureCore/CaptureClient.swift
import Foundation
enum RoundTrip: Equatable, Sendable {
case text(String)
case timeout
case disconnected
}
enum CaptureConnectionError: LocalizedError {
case timedOut
case closed
var errorDescription: String? {
switch self {
case .timedOut:
return "サーバに接続できませんでした"
case .closed:
return "接続が閉じました"
}
}
}
/// `ws://127.0.0.1:3000/capture` を維持し、PNG をバイナリ1通で送ってテキスト応答を待つ。
actor CaptureClient: FrameSending {
private let url: URL
private let logger: any CaptureLogging
private let responseTimeout: TimeInterval
private let openTimeout: TimeInterval
private var session: URLSession?
private var task: URLSessionWebSocketTask?
private var bridge: ConnectionBridge?
private var receiveTask: Task<Void, Never>?
private var generation = 0
private var opened = false
private var waitingForResponse = false
private var buffered: String?
private var waiter: CheckedContinuation<RoundTrip, Never>?
private var waitID: UUID?
init(
url: URL,
logger: any CaptureLogging,
responseTimeout: TimeInterval = 20,
openTimeout: TimeInterval = 5
) {
self.url = url
self.logger = logger
self.responseTimeout = responseTimeout
self.openTimeout = openTimeout
}
var isOpen: Bool {
opened
}
func connect() async throws {
closeQuietly()
let generation = self.generation
let bridge = ConnectionBridge()
let configuration = URLSessionConfiguration.ephemeral
configuration.waitsForConnectivity = false
let session = URLSession(configuration: configuration, delegate: bridge, delegateQueue: nil)
let task = session.webSocketTask(with: url)
task.maximumMessageSize = 16 * 1024 * 1024
self.bridge = bridge
self.session = session
self.task = task
task.resume()
do {
try await bridge.waitForOpen(timeout: openTimeout)
} catch {
closeQuietly()
throw error
}
opened = true
receiveTask = Task { [weak self] in
await self?.receiveLoop(generation: generation, task: task)
}
}
func roundTrip(_ png: Data) async -> RoundTrip {
guard opened, let task else {
return .disconnected
}
let id = UUID()
buffered = nil
waitingForResponse = true
do {
// テキストフレームは送らない。中身は PNG のバイトそのもの。
try await task.send(.data(png))
} catch {
opened = false
waitingForResponse = false
buffered = nil
return .disconnected
}
if let text = buffered {
buffered = nil
waitingForResponse = false
return .text(text)
}
if !opened {
waitingForResponse = false
return .disconnected
}
return await withCheckedContinuation { continuation in
if let text = buffered {
buffered = nil
waitingForResponse = false
continuation.resume(returning: .text(text))
return
}
if !opened {
waitingForResponse = false
continuation.resume(returning: .disconnected)
return
}
waiter = continuation
waitID = id
let timeout = responseTimeout
Task {
try? await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000))
self.fireTimeout(id)
}
}
}
func close() {
closeQuietly()
}
private func receiveLoop(generation: Int, task: URLSessionWebSocketTask) async {
while !Task.isCancelled, self.generation == generation {
do {
let message = try await task.receive()
guard self.generation == generation else { return }
guard let text = Self.text(from: message) else { continue }
accept(text)
} catch {
guard self.generation == generation else { return }
opened = false
failWaiter(.disconnected)
return
}
}
}
private func accept(_ text: String) {
if let waiter {
self.waiter = nil
waitID = nil
waitingForResponse = false
buffered = nil
waiter.resume(returning: .text(text))
return
}
if waitingForResponse {
buffered = text
return
}
// 待ちを打ち切ったあとに届いた応答は、次の画像の結果にしない。
logger.log("遅れて届いた応答: \(describeServerMessage(parseServerMessage(text)))")
}
private func fireTimeout(_ id: UUID) {
guard waitID == id else { return }
waitID = nil
waitingForResponse = false
guard let waiter else { return }
self.waiter = nil
waiter.resume(returning: .timeout)
}
private func failWaiter(_ outcome: RoundTrip) {
waitingForResponse = false
buffered = nil
waitID = nil
guard let waiter else { return }
self.waiter = nil
waiter.resume(returning: outcome)
}
private func closeQuietly() {
generation += 1
opened = false
failWaiter(.disconnected)
receiveTask?.cancel()
receiveTask = nil
task?.cancel(with: .goingAway, reason: nil)
session?.invalidateAndCancel()
task = nil
session = nil
bridge = nil
}
private static func text(from message: URLSessionWebSocketTask.Message) -> String? {
switch message {
case .string(let text):
return text
case .data(let data):
return String(data: data, encoding: .utf8)
@unknown default:
return nil
}
}
}
private final class ConnectionBridge: NSObject, URLSessionWebSocketDelegate, @unchecked Sendable {
private let lock = NSLock()
private var continuation: CheckedContinuation<Void, Error>?
private var settled = false
func waitForOpen(timeout: TimeInterval) async throws {
try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
self.lock.lock()
self.continuation = continuation
self.lock.unlock()
DispatchQueue.global().asyncAfter(deadline: .now() + timeout) { [weak self] in
self?.settle(.failure(CaptureConnectionError.timedOut))
}
}
}
func urlSession(
_ session: URLSession,
webSocketTask: URLSessionWebSocketTask,
didOpenWithProtocol protocol: String?
) {
settle(.success(()))
}
func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
if let error {
settle(.failure(error))
}
}
private func settle(_ result: Result<Void, Error>) {
lock.lock()
guard !settled, let continuation else {
lock.unlock()
return
}
settled = true
self.continuation = nil
lock.unlock()
switch result {
case .success:
continuation.resume()
case .failure(let error):
continuation.resume(throwing: error)
}
}
}
サーバはポート 3000 で待ち、/capture を WebSocket に上げます。届いたバイトを相手認識へ渡し、成功なら6体の配列、失敗なら { "code": "..." } をテキストで返します。
pokemon-champions-api/src/server.ts
import { serve } from "@hono/node-server";
import { WebSocketServer } from "ws";
import { createApp } from "./app";
const app = createApp();
const port = 3000;
serve(
{
fetch: app.fetch,
port,
websocket: { server: new WebSocketServer({ noServer: true }) },
},
(info) => {
console.log(`http://localhost:${info.port}`);
},
);
createApp が相手認識を作り、ソケットへ渡します。図鑑に種族名があるかも、ここで確認します。
pokemon-champions-api/src/app.ts
import { RecognizeOpponentParty } from "./module/selection-board/application/query/recognize-opponent-party";
import { registerCaptureSocket } from "./module/selection-board/controller/ws/capture-socket";
pokemon-champions-api/src/app.ts
let pokemonTools: ReturnType<typeof createPokemonTools> | undefined;
const tools = () => (pokemonTools ??= createPokemonTools());
const judgeMoveEffect = async (input: JudgeMoveEffectInput) => tools().judgeMoveEffect(input);
const recognizeOpponentParty = new RecognizeOpponentParty(
(input) => tools().recognizeOpponent(input),
async (name) => (await speciesRepository.getByName(name)) !== null,
);
pokemon-champions-api/src/app.ts
registerCaptureSocket(app, recognizeOpponentParty);
図鑑アイコンの場所を決め、画像認識を呼びます。
pokemon-champions-api/src/module/pokemon-tools/provided/pokemon-tools.ts
const moduleDir = path.dirname(fileURLToPath(import.meta.url));
const catalogSourceCandidates = [
path.resolve(moduleDir, "../../../../../pokemon-champions/data/source"),
path.resolve(moduleDir, "../../../../.symboliclink/pokemon_champions/data/source"),
];
function catalogSourceReady(dir: string): boolean {
const iconDir = path.join(dir, "champions_image");
const pokedexPath = path.join(dir, "champions_pokedex.csv");
if (!fs.existsSync(pokedexPath) || !fs.existsSync(iconDir)) return false;
if (!fs.statSync(iconDir).isDirectory()) return false;
return fs.readdirSync(iconDir).some((file) => file.endsWith(".png"));
}
/** 図鑑の data/source。アイコンと台帳がある場所を返し、無いときは隣の pokemon-champions を返す。 */
export function defaultCatalogSourceDir(): string {
return catalogSourceCandidates.find((dir) => catalogSourceReady(dir)) ?? catalogSourceCandidates[0];
}
createPokemonTools が、その場所を渡して recognizeOpponentImage を呼びます。
pokemon-champions-api/src/module/pokemon-tools/provided/pokemon-tools.ts
/** createPokemonTools は、Jev クライアントを内側で組み立てて PokemonTools を返す。 */
export function createPokemonTools(options: CreatePokemonToolsOptions = {}): PokemonTools {
const jev = options.jev ?? createJev({ apiKey: options.apiKey, model: options.model });
const statusAilment = createStatusAilmentJudge(jev);
const flinch = createFlinchJudge(jev);
const confusion = createConfusionJudge(jev);
const statChange = createStatChangeJudge(jev);
const hpRecovery = createHpRecoveryJudge(jev);
const recoil = createRecoilJudge(jev);
const itemManipulation = createItemManipulationJudge(jev);
const switchPokemon = createSwitchJudge(jev);
const fieldChange = createFieldChangeJudge(jev);
const specialEffects = createSpecialEffectsJudge(jev);
const lockedAction = createLockedActionJudge(jev);
const sourceDir = defaultCatalogSourceDir();
const iconDir = options.iconDir ?? path.join(sourceDir, "champions_image");
const pokedexPath = options.pokedexPath ?? path.join(sourceDir, "champions_pokedex.csv");
return {
askSimpleExample: () => runAskSimpleExample(jev),
recognizeOpponent: (input) =>
recognizeOpponentImage({
image: input.image,
topk: input.topk,
iconDir,
pokedexPath,
}),
judgeMoveEffect: (input) =>
runJudgeMoveEffect(
{
statusAilment,
flinch,
confusion,
statChange,
hpRecovery,
recoil,
itemManipulation,
switchPokemon,
fieldChange,
specialEffects,
lockedAction,
},
input,
options.random,
),
};
}
pokemon-champions-api/src/module/selection-board/controller/ws/capture-socket.ts
import { upgradeWebSocket } from "@hono/node-server";
import type { Hono } from "hono";
import type { WSMessageReceive } from "hono/ws";
import type { ConfirmedOpponentSlot, RecognizeOpponentParty } from "../../application/query/recognize-opponent-party";
export type CaptureFeed = {
slots: ConfirmedOpponentSlot[] | null;
};
async function imageBytes(data: WSMessageReceive): Promise<Buffer> {
if (typeof data === "string") return Buffer.from(data);
if (data instanceof ArrayBuffer) return Buffer.copyBytesFrom(new Uint8Array(data));
if (data instanceof Blob) return Buffer.copyBytesFrom(new Uint8Array(await data.arrayBuffer()));
return Buffer.copyBytesFrom(Buffer.from(data));
}
/** PNG を既存の相手認識へ渡し、6体の JSON を返す。成功した最新を同じプロセスの GET が読む。 */
export function registerCaptureSocket(
app: Hono,
query: RecognizeOpponentParty,
feed: CaptureFeed = { slots: null },
): CaptureFeed {
let seen = 0;
let stored = 0;
app.get("/capture/opponents", (c) => c.json(feed.slots ?? []));
app.get(
"/capture",
upgradeWebSocket(() => ({
async onMessage(event, ws) {
const mine = ++seen;
try {
const result = await query.execute({ image: await imageBytes(event.data) });
if (result.ok && mine > stored) {
stored = mine;
feed.slots = result.slots.slice();
}
ws.send(JSON.stringify(result.ok ? result.slots : { code: result.code }));
} catch {
// 認識の例外や、閉じた接続への送信ではプロセスを落とさない。
}
},
})),
);
return feed;
}
認識結果の種族名を、このAPIの図鑑にあるか枠ごとに確認します。無い枠は speciesName を null にし、review と species_not_found を付けて残します。
pokemon-champions-api/src/module/selection-board/application/query/recognize-opponent-party.ts
import type { PokemonTools } from "../../../pokemon-tools/provided/pokemon-tools";
export type RecognizeOpponent = PokemonTools["recognizeOpponent"];
export type SpeciesNameLookup = (speciesName: string) => Promise<boolean>;
type RecognizedSlot = Extract<Awaited<ReturnType<RecognizeOpponent>>, { ok: true }>["slots"][number];
export type ConfirmedOpponentSlot = Omit<RecognizedSlot, "speciesName" | "warnings"> & {
speciesName: string | null;
warnings: Array<RecognizedSlot["warnings"][number] | "species_not_found">;
};
export type RecognizeOpponentPartyResult =
| { ok: true; slots: ConfirmedOpponentSlot[] }
| { ok: false; code: "invalid_input" | "selection_screen_not_found" | "templates_unavailable" };
/** 認識結果の種族名を、この API の図鑑にあるか確認する。先後は計算しない。 */
export class RecognizeOpponentParty {
constructor(
private readonly recognize: RecognizeOpponent,
private readonly speciesExists: SpeciesNameLookup,
) {}
async execute(input: { image: Buffer; topk?: number }): Promise<RecognizeOpponentPartyResult> {
const recognized = await this.recognize(input);
if (!recognized.ok) return recognized;
const slots: ConfirmedOpponentSlot[] = [];
for (const slot of recognized.slots) {
if (await this.speciesExists(slot.speciesName)) {
slots.push(slot);
continue;
}
const warnings: ConfirmedOpponentSlot["warnings"] = [...slot.warnings, "species_not_found"];
slots.push({
...slot,
speciesName: null,
status: "review",
warnings,
});
}
return { ok: true, slots };
}
}
③ OpenCVで相手6体を解析
サーバで受信した画像をOpenCVで解析します。
処理の流れは以下のとおりです。
- 選出画面の右側にある相手6体の表示領域を検出
- 6つの枠からポケモン画像を切り出す
- 図鑑アイコンと照合してポケモンを特定
画像比較には、正規化相関・色ヒストグラム・エッジの情報を組み合わせています。認識スコアが低い場合や候補間の差が小さい場合は、確認が必要な結果として扱います。
詳しい画像解析の仕組みは過去記事で説明しています。実装は次のとおりです。
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/application/recognize-opponent.ts
import { evaluateConfidence, type ConfidenceWarning } from "../domain/confidence";
import { matchScore, rankMatches, type MatchFeatures } from "../domain/score-match";
import { splitOpponentSlots } from "../domain/slot-geometry";
import { detectOpponentBand } from "../infrastructure/detect-slots";
import { loadImageAsBgrMat } from "../infrastructure/image-features";
import { getCv } from "../infrastructure/opencv";
import { loadTemplateBank, spriteFeatures, type CatalogIcon } from "../infrastructure/template-bank";
export type RecognizedCandidate = {
speciesName: string;
displayName: string;
nationalDexNo: number;
score: number;
};
export type RecognizedSlot = {
slot: number;
speciesName: string;
displayName: string;
nationalDexNo: number;
score: number;
status: "ok" | "review";
warnings: ConfidenceWarning[];
candidates: RecognizedCandidate[];
};
export type RecognizeOpponentImageInput = {
image: Buffer;
topk?: number;
iconDir: string;
pokedexPath: string;
};
export type RecognizeOpponentFailure = "invalid_input" | "selection_screen_not_found" | "templates_unavailable";
export type RecognizeOpponentImageResult =
| { ok: true; slots: RecognizedSlot[] }
| { ok: false; code: RecognizeOpponentFailure };
function roundedMargin(first: number, second: number | undefined): number | null {
if (second === undefined) return null;
const margin = Math.round((first - second) * 1000) / 1000;
return margin === 0 ? 0 : margin;
}
/** 選出画面の画像から、相手の6枠を図鑑アイコンと照合する。 */
export async function recognizeOpponentImage(input: RecognizeOpponentImageInput): Promise<RecognizeOpponentImageResult> {
const topk = input.topk ?? 3;
if (!Number.isInteger(topk) || topk < 1) return { ok: false, code: "invalid_input" };
let mat: { matBgr: any; cleanup: () => void };
try {
const cv = await getCv();
mat = await loadImageAsBgrMat(input.image, cv);
try {
const detected = detectOpponentBand(mat.matBgr, cv);
if (!detected.ok) return detected;
const split = splitOpponentSlots(detected.band);
if (!split.ok) return { ok: false, code: "selection_screen_not_found" };
const bank = await loadTemplateBank(input.iconDir, input.pokedexPath, cv);
if (bank.length === 0) return { ok: false, code: "templates_unavailable" };
const byFile = new Map(bank.map((icon) => [icon.imageFile, icon]));
const slots: RecognizedSlot[] = [];
for (let index = 0; index < split.slots.length; index++) {
const geometry = split.slots[index];
if (!geometry) continue;
const features = spriteFeatures(mat.matBgr, geometry.card, geometry.sprite, bank, cv);
const bestBySpecies = new Map<string, MatchFeatures>();
for (const match of features) {
const icon = byFile.get(match.id);
const key = icon?.speciesName ?? match.id;
const prev = bestBySpecies.get(key);
if (!prev || matchScore(match) > matchScore(prev)) {
bestBySpecies.set(key, match);
}
}
const ranked = rankMatches(Array.from(bestBySpecies.values()), topk);
if (!ranked.ok) return ranked;
const top = ranked.matches[0];
const icon = top ? byFile.get(top.id) : undefined;
if (!top || !icon) return { ok: false, code: "templates_unavailable" };
const confidence = evaluateConfidence({
score: top.score,
margin: roundedMargin(top.score, ranked.matches[1]?.score),
});
slots.push({
slot: index + 1,
speciesName: icon.speciesName,
displayName: icon.displayName,
nationalDexNo: icon.nationalDexNo,
score: top.score,
status: confidence.status,
warnings: confidence.warnings,
candidates: ranked.matches.flatMap((match) => {
const candidate = byFile.get(match.id);
return candidate ? [toCandidate(candidate, match.score)] : [];
}),
});
}
return { ok: true, slots };
} finally {
mat.cleanup();
}
} catch {
return { ok: false, code: "invalid_input" };
}
}
function toCandidate(icon: CatalogIcon, score: number): RecognizedCandidate {
return {
speciesName: icon.speciesName,
displayName: icon.displayName,
nationalDexNo: icon.nationalDexNo,
score,
};
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/infrastructure/opencv.ts
let cvInstance: any = null;
let cvPromise: Promise<any> | null = null;
/** OpenCV.js を1回だけ起動する。 */
export async function getCv(): Promise<any> {
if (cvInstance) return cvInstance;
if (!cvPromise) {
cvPromise = (async () => {
const cvModule = (await import("@techstark/opencv-js")) as { default?: unknown };
const loaded: unknown = cvModule.default ?? cvModule;
cvInstance =
loaded && typeof loaded === "object" && "then" in loaded
? await (loaded as Promise<unknown>)
: loaded;
return cvInstance;
})();
}
return cvPromise;
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/infrastructure/recognition-config.ts
export const FEATURE_SIZE = 96;
/** 相関へ進む種族数。色の上位と輪郭の上位を、それぞれこの数だけ残す。 */
export const SPECIES_SHORTLIST = 15;
export const MASK_THRESHOLD = 40;
export const NCC_MIN_PIXELS = 80;
export const HIST_H_BINS = 18;
export const HIST_S_BINS = 16;
export const RIGHT_ROI_RATIO = 0.62;
export const CARD_HSV_LO: [number, number, number] = [165, 170, 60];
export const CARD_HSV_HI: [number, number, number] = [179, 255, 185];
export const SPRITE_BG_HSV_LO: [number, number, number] = [160, 70, 28];
export const SPRITE_BG_HSV_HI: [number, number, number] = [179, 255, 235];
export const BG_RGB_DISTANCE = 42;
/** カードの緑よりこれ以上大きい画素は、赤紫に照らされた本体として残す。 */
export const CARD_BG_GREEN_MARGIN = 26;
/** カード色よりこれ以上明るく、彩度がカードほど高くない画素も本体として残す。 */
export const CARD_BG_VALUE_MARGIN = 45;
export const CARD_BG_MAX_SATURATION = 180;
/** 元のシルエットからこの距離以内の画素だけ、赤紫マスクから本体へ戻す。 */
export const SPRITE_RESTORE_RADIUS = 16;
/** これ以上本体が残っている枠は、カード側の画素を戻さない。 */
export const SPRITE_RESTORE_MAX_COVERAGE = 0.125;
/** 枠の右端はタイプアイコンなので、戻す対象から外す。 */
export const SPRITE_RESTORE_RIGHT_LIMIT = 0.82;
export const FALLBACK_CARD_BGR: [number, number, number] = [70, 15, 140];
export const MIN_BAND_WIDTH = 80;
export const MIN_BAND_HEIGHT = 200;
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/infrastructure/detect-slots.ts
import { dropShortHeader } from "../domain/band-rows";
import type { Box } from "../domain/slot-geometry";
import {
CARD_HSV_HI,
CARD_HSV_LO,
MIN_BAND_HEIGHT,
MIN_BAND_WIDTH,
RIGHT_ROI_RATIO,
} from "./recognition-config";
export type DetectBandResult = { ok: true; band: Box } | { ok: false; code: "selection_screen_not_found" };
function percentile(values: Float32Array, p: number): number {
if (values.length === 0) return 0;
const sorted = Array.from(values).sort((a, b) => a - b);
return sorted[Math.min(sorted.length - 1, Math.floor((sorted.length * p) / 100))];
}
function convolveSame9(values: Float32Array): Float32Array {
const out = new Float32Array(values.length);
for (let i = 0; i < values.length; i++) {
let sum = 0;
for (let k = -4; k <= 4; k++) {
const idx = i + k;
if (idx >= 0 && idx < values.length) sum += values[idx];
}
out[i] = sum / 9;
}
return out;
}
function fillSmallGaps(active: Uint8Array, maxGap: number): boolean[] {
const filled = Array.from(active, (value) => value === 1);
let i = 0;
while (i < filled.length) {
if (filled[i]) {
i++;
continue;
}
let j = i;
while (j < filled.length && !filled[j]) j++;
if (i > 0 && j < filled.length && j - i <= maxGap) {
for (let k = i; k < j; k++) filled[k] = true;
}
i = j;
}
return filled;
}
function longestRun(active: ArrayLike<boolean | number>): [number, number] {
let best: [number, number] = [0, 0];
let i = 0;
while (i < active.length) {
if (!active[i]) {
i++;
continue;
}
let j = i;
while (j < active.length && active[j]) j++;
if (j - i > best[1] - best[0]) best = [i, j];
i = j;
}
return best;
}
/** 画面右側の赤紫の帯を、6枠へ切る前の矩形として返す。 */
export function detectOpponentBand(imageBgr: any, cv: any): DetectBandResult {
const height = imageBgr.rows;
const width = imageBgr.cols;
const xOrigin = Math.floor(width * RIGHT_ROI_RATIO);
const roiRect = new cv.Rect(xOrigin, 0, width - xOrigin, height);
const roiSub = imageBgr.roi(roiRect);
const roi = roiSub.clone();
roiSub.delete();
const hsv = new cv.Mat();
cv.cvtColor(roi, hsv, cv.COLOR_BGR2HSV);
const lowCard = new cv.Mat(roi.rows, roi.cols, cv.CV_8UC3, [...CARD_HSV_LO, 0]);
const highCard = new cv.Mat(roi.rows, roi.cols, cv.CV_8UC3, [...CARD_HSV_HI, 0]);
const mask = new cv.Mat();
cv.inRange(hsv, lowCard, highCard, mask);
const rawMask = mask.clone();
const kernel7 = cv.Mat.ones(7, 7, cv.CV_8U);
cv.morphologyEx(mask, mask, cv.MORPH_CLOSE, kernel7);
const release = () => {
lowCard.delete();
highCard.delete();
mask.delete();
rawMask.delete();
kernel7.delete();
hsv.delete();
roi.delete();
};
const maskData = mask.data;
const colSum = new Float32Array(roi.cols);
for (let y = 0; y < roi.rows; y++) {
const rowOffset = y * roi.cols;
for (let x = 0; x < roi.cols; x++) {
if (maskData[rowOffset + x] > 0) colSum[x]++;
}
}
let maxCol = 0;
for (let x = 0; x < roi.cols; x++) if (colSum[x] > maxCol) maxCol = colSum[x];
if (maxCol < 40) {
release();
return { ok: false, code: "selection_screen_not_found" };
}
const denseThresh = Math.max(40, maxCol * 0.18);
const dense = new Uint8Array(roi.cols);
for (let x = 0; x < roi.cols; x++) if (colSum[x] > denseThresh) dense[x] = 1;
const [xL, xR] = longestRun(fillSmallGaps(dense, 25));
if (xR - xL < MIN_BAND_WIDTH) {
release();
return { ok: false, code: "selection_screen_not_found" };
}
const rowSum = new Float32Array(roi.rows);
for (let y = 0; y < roi.rows; y++) {
const rowOffset = y * roi.cols;
for (let x = xL; x < xR; x++) if (maskData[rowOffset + x] > 0) rowSum[y]++;
}
const smooth = convolveSame9(rowSum);
const activeThresh = Math.max(18, percentile(rowSum, 55) * 0.5);
const active = new Uint8Array(roi.rows);
for (let y = 0; y < roi.rows; y++) if (smooth[y] > activeThresh) active[y] = 1;
let [yT, yB] = longestRun(fillSmallGaps(active, 24));
const bandW = xR - xL;
for (let y = yT; y < Math.min(yB, yT + 40); y++) {
if (rowSum[y] < 5) {
let start = y;
while (start < yB && rowSum[start] < bandW * 0.5) start++;
if (start < yB && start - yT <= 40) {
yT = start;
break;
}
}
}
const rawRows = new Float32Array(roi.rows);
const rawData = rawMask.data;
for (let y = 0; y < roi.rows; y++) {
const rowOffset = y * roi.cols;
for (let x = xL; x < xR; x++) if (rawData[rowOffset + x] > 0) rawRows[y]++;
}
yT = dropShortHeader(rawRows, yT, yB, bandW);
release();
if (yB - yT < MIN_BAND_HEIGHT) return { ok: false, code: "selection_screen_not_found" };
return { ok: true, band: [xOrigin + xL, yT, xOrigin + xR, yB] };
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/domain/band-rows.ts
/** 帯の先頭で、カード全幅に満たない短い名前プレートを外す。 */
export function dropShortHeader(
rowCounts: ArrayLike<number>,
start: number,
end: number,
width: number,
): number {
if (width <= 0 || end <= start) return start;
const strong = width * 0.7;
let y = start;
while (y < end && (rowCounts[y] ?? 0) < strong) y++;
const prefix = y - start;
if (prefix >= 8 && prefix <= 40 && y < end) return y;
return start;
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/domain/slot-geometry.ts
export type Box = readonly [number, number, number, number];
export type OpponentSlotGeometry = {
card: Box;
sprite: Box;
};
export type SplitOpponentSlotsResult =
| { ok: true; slots: OpponentSlotGeometry[] }
| { ok: false; code: "invalid_input" };
const SLOT_COUNT = 6;
const SPRITE_INSET_PX = 2;
const SPRITE_WIDTH_RATIO = 0.64;
/** 検出済みのカード帯を、上から6枚のカードと照合範囲に分ける。 */
export function splitOpponentSlots(band: Box): SplitOpponentSlotsResult {
const [x0, y0, x1, y1] = band;
const width = x1 - x0;
const height = y1 - y0;
if (width <= 0 || height <= 0) {
return { ok: false, code: "invalid_input" };
}
const insetX = Math.max(4, Math.floor(width * 0.03));
const slots: OpponentSlotGeometry[] = [];
for (let i = 0; i < SLOT_COUNT; i++) {
const sy0 = y0 + Math.floor((height * i) / SLOT_COUNT);
const sy1 = y0 + Math.floor((height * (i + 1)) / SLOT_COUNT);
const gap = Math.max(2, Math.floor((sy1 - sy0) * 0.04));
const card: Box = [x0 + insetX, sy0 + gap, x1 - insetX, sy1 - gap];
slots.push({ card, sprite: spriteRegion(card) });
}
return { ok: true, slots };
}
function spriteRegion(card: Box): Box {
const [x0, y0, x1, y1] = card;
const width = x1 - x0;
return [x0 + SPRITE_INSET_PX, y0 + SPRITE_INSET_PX, x0 + Math.floor(width * SPRITE_WIDTH_RATIO), y1 - SPRITE_INSET_PX];
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/infrastructure/template-bank.ts
import fs from "node:fs";
import path from "node:path";
import sharp from "sharp";
import type { MatchFeatures } from "../domain/score-match";
import { shortlistSpecies } from "../domain/shortlist-species";
import {
cropBox,
edgeMap,
estimateCardBg,
extractQuery,
hsHist,
letterbox,
bestMaskedNcc,
opaqueBbox,
vectorDot,
type IconFeatures,
} from "./image-features";
import { FEATURE_SIZE, SPECIES_SHORTLIST } from "./recognition-config";
import type { Box } from "../domain/slot-geometry";
export type CatalogIcon = {
imageFile: string;
speciesName: string;
displayName: string;
nationalDexNo: number;
};
export type IconTemplate = CatalogIcon & IconFeatures;
const CSV_COLUMNS = ["id", "name_ja", "species_name_ja", "api_name", "species_id", "is_mega", "image_file"] as const;
export function readPokedex(csvPath: string): Map<string, CatalogIcon> {
const byFile = new Map<string, CatalogIcon>();
if (!fs.existsSync(csvPath)) return byFile;
const lines = fs.readFileSync(csvPath, "utf8").split(/\r?\n/).filter((line) => line.length > 0);
const header = lines[0]?.split(",") ?? [];
const index = Object.fromEntries(CSV_COLUMNS.map((column) => [column, header.indexOf(column)]));
if (CSV_COLUMNS.some((column) => index[column] < 0)) return byFile;
for (const line of lines.slice(1)) {
const cells = line.split(",");
const imageFile = cells[index.image_file] ?? "";
const speciesName = cells[index.api_name] ?? "";
const displayName = cells[index.name_ja] ?? "";
const nationalDexNo = Number(cells[index.species_id]);
if (!imageFile || !speciesName || !displayName || !Number.isInteger(nationalDexNo)) continue;
byFile.set(imageFile, { imageFile, speciesName, displayName, nationalDexNo });
}
return byFile;
}
function pngFiles(directory: string): string[] {
if (!fs.existsSync(directory) || !fs.statSync(directory).isDirectory()) return [];
return fs.readdirSync(directory).filter((file) => file.endsWith(".png")).sort();
}
async function templateFromIcon(imagePath: string, icon: CatalogIcon, cv: any): Promise<IconTemplate | null> {
let raw: { data: Buffer; info: sharp.OutputInfo };
try {
raw = await sharp(imagePath).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
} catch {
return null;
}
const { width, height, channels } = raw.info;
if (channels < 4) return null;
const alpha = new Uint8Array(width * height);
for (let i = 0; i < alpha.length; i++) alpha[i] = raw.data[i * 4 + 3];
const box = opaqueBbox(alpha, width, height, 50);
if (!box) return null;
const [x0, y0, x1, y1] = box;
const cropW = x1 - x0;
const cropH = y1 - y0;
const cropBgr = new cv.Mat(cropH, cropW, cv.CV_8UC3);
const cropMask = new cv.Mat(cropH, cropW, cv.CV_8UC1);
try {
for (let cy = 0; cy < cropH; cy++) {
for (let cx = 0; cx < cropW; cx++) {
const srcIdx = ((y0 + cy) * width + x0 + cx) * 4;
const dst = cy * cropW + cx;
cropBgr.data[dst * 3] = raw.data[srcIdx + 2];
cropBgr.data[dst * 3 + 1] = raw.data[srcIdx + 1];
cropBgr.data[dst * 3 + 2] = raw.data[srcIdx];
cropMask.data[dst] = raw.data[srcIdx + 3];
}
}
const { bgrCanvas, maskCanvas } = letterbox(cropBgr, cropMask, cv, FEATURE_SIZE);
try {
return {
...icon,
rgb: new Float32Array(bgrCanvas.data),
mask: new Uint8Array(maskCanvas.data),
hist: hsHist(bgrCanvas, maskCanvas, cv),
edge: edgeMap(bgrCanvas, maskCanvas, cv),
};
} finally {
bgrCanvas.delete();
maskCanvas.delete();
}
} finally {
cropBgr.delete();
cropMask.delete();
}
}
function isBattleOnlyForm(speciesName: string): boolean {
if (speciesName.includes("-mega")) return true;
if (speciesName.startsWith("castform-") && speciesName !== "castform") return true;
if (speciesName === "aegislash-blade") return true;
if (speciesName.includes("-zen")) return true;
if (speciesName === "palafin-hero") return true;
// えいえんのはなは種族名ではなく、欠けたシルエットに高得点で当たる。
if (speciesName === "floette-eternal") return true;
return false;
}
const bankCache = new Map<string, IconTemplate[]>();
export async function loadTemplateBank(iconDir: string, pokedexPath: string, cv: any): Promise<IconTemplate[]> {
const cacheKey = `${iconDir}:${pokedexPath}`;
const cached = bankCache.get(cacheKey);
if (cached) return cached;
const pokedex = readPokedex(pokedexPath);
const templates: IconTemplate[] = [];
for (const file of pngFiles(iconDir)) {
const icon = pokedex.get(file);
if (!icon || isBattleOnlyForm(icon.speciesName)) continue;
const template = await templateFromIcon(path.join(iconDir, file), icon, cv);
if (template) templates.push(template);
}
const shinyDir = path.join(iconDir, "shiny");
if (fs.existsSync(shinyDir) && fs.statSync(shinyDir).isDirectory()) {
for (const file of pngFiles(shinyDir)) {
const icon = pokedex.get(file);
if (!icon || isBattleOnlyForm(icon.speciesName)) continue;
const template = await templateFromIcon(path.join(shinyDir, file), {
...icon,
imageFile: `shiny/${file}`,
}, cv);
if (template) templates.push(template);
}
}
bankCache.set(cacheKey, templates);
return templates;
}
export function featuresForSprite(cropBgr: any, bgBgr: [number, number, number], bank: readonly IconTemplate[], cv: any): MatchFeatures[] {
const { queryRgb, queryMask } = extractQuery(cropBgr, bgBgr, cv);
const queryBgr = new cv.Mat(FEATURE_SIZE, FEATURE_SIZE, cv.CV_8UC3);
const queryMaskMat = new cv.Mat(FEATURE_SIZE, FEATURE_SIZE, cv.CV_8UC1);
queryBgr.data.set(queryRgb);
queryMaskMat.data.set(queryMask);
const queryHist = hsHist(queryBgr, queryMaskMat, cv);
const queryEdge = edgeMap(queryBgr, queryMaskMat, cv);
queryBgr.delete();
queryMaskMat.delete();
const scored = bank.map((template) => ({
template,
hist: vectorDot(template.hist, queryHist),
edge: vectorDot(template.edge, queryEdge),
}));
const kept = new Set(shortlistSpecies(
scored.map((row) => ({ speciesName: row.template.speciesName, hist: row.hist, edge: row.edge })),
SPECIES_SHORTLIST,
));
return scored.flatMap((row) => {
if (!kept.has(row.template.speciesName)) return [];
return [{
id: row.template.imageFile,
ncc: bestMaskedNcc(queryRgb, queryMask, row.template.rgb, row.template.mask, row.hist),
hist: row.hist,
edge: row.edge,
}];
});
}
export function spriteFeatures(imageBgr: any, card: Box, sprite: Box, bank: readonly IconTemplate[], cv: any): MatchFeatures[] {
const spriteCrop = cropBox(imageBgr, sprite, cv);
const cardCrop = cropBox(imageBgr, card, cv);
try {
return featuresForSprite(spriteCrop, estimateCardBg(cardCrop, cv), bank, cv);
} finally {
spriteCrop.delete();
cardCrop.delete();
}
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/domain/shortlist-species.ts
export type SpeciesSimilarity = {
speciesName: string;
hist: number;
edge: number;
};
type SpeciesBest = SpeciesSimilarity & {
index: number;
};
/**
* 色の上位と輪郭の上位の和を、相関へ進む種族として返す。
* 同じ種族は色の最大と輪郭の最大を使う。色の順位を先に並べ、輪郭だけで入った種族をその後に足す。
* 同点は、入力で先に出た種族を上位にする。
*/
export function shortlistSpecies(rows: readonly SpeciesSimilarity[], limit: number): string[] {
if (!Number.isInteger(limit) || limit < 1) return [];
const bySpecies = new Map<string, SpeciesBest>();
for (let index = 0; index < rows.length; index++) {
const row = rows[index];
if (!row) continue;
const prev = bySpecies.get(row.speciesName);
if (!prev) {
bySpecies.set(row.speciesName, { speciesName: row.speciesName, hist: row.hist, edge: row.edge, index });
continue;
}
if (row.hist > prev.hist) prev.hist = row.hist;
if (row.edge > prev.edge) prev.edge = row.edge;
}
const species = Array.from(bySpecies.values());
const selected: string[] = [];
const seen = new Set<string>();
for (const item of ranked(species, "hist").slice(0, limit)) {
selected.push(item.speciesName);
seen.add(item.speciesName);
}
for (const item of ranked(species, "edge").slice(0, limit)) {
if (seen.has(item.speciesName)) continue;
selected.push(item.speciesName);
seen.add(item.speciesName);
}
return selected;
}
function ranked(species: SpeciesBest[], key: "hist" | "edge"): SpeciesBest[] {
return [...species].sort((left, right) => {
if (left[key] !== right[key]) return right[key] - left[key];
return left.index - right.index;
});
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/infrastructure/image-features.ts
import sharp from "sharp";
import type { Box } from "../domain/slot-geometry";
import {
BG_RGB_DISTANCE,
CARD_BG_GREEN_MARGIN,
CARD_BG_MAX_SATURATION,
CARD_BG_VALUE_MARGIN,
FALLBACK_CARD_BGR,
FEATURE_SIZE,
HIST_H_BINS,
HIST_S_BINS,
MASK_THRESHOLD,
NCC_MIN_PIXELS,
SPRITE_BG_HSV_HI,
SPRITE_BG_HSV_LO,
SPRITE_RESTORE_MAX_COVERAGE,
SPRITE_RESTORE_RADIUS,
SPRITE_RESTORE_RIGHT_LIMIT,
} from "./recognition-config";
export type IconFeatures = {
rgb: Float32Array;
mask: Uint8Array;
hist: Float32Array;
edge: Float32Array;
};
export async function loadImageAsBgrMat(input: Buffer, cv: any): Promise<{ matBgr: any; cleanup: () => void }> {
const { data, info } = await sharp(input).removeAlpha().raw().toBuffer({ resolveWithObject: true });
const matRgb = new cv.Mat(info.height, info.width, cv.CV_8UC3);
matRgb.data.set(data);
const matBgr = new cv.Mat();
cv.cvtColor(matRgb, matBgr, cv.COLOR_RGB2BGR);
matRgb.delete();
return { matBgr, cleanup: () => matBgr.delete() };
}
export function cropBox(image: any, box: Box, cv: any): any {
const [x0, y0, x1, y1] = box;
const rect = new cv.Rect(x0, y0, Math.max(1, x1 - x0), Math.max(1, y1 - y0));
const roi = image.roi(rect);
const cloned = roi.clone();
roi.delete();
return cloned;
}
export function opaqueBbox(mask: Uint8Array, width: number, height: number, minPixels: number): Box | null {
let minX = width;
let maxX = -1;
let minY = height;
let maxY = -1;
let count = 0;
for (let y = 0; y < height; y++) {
const rowOffset = y * width;
for (let x = 0; x < width; x++) {
if (mask[rowOffset + x] > 32) {
count++;
if (x < minX) minX = x;
if (x > maxX) maxX = x;
if (y < minY) minY = y;
if (y > maxY) maxY = y;
}
}
}
if (count < minPixels) return null;
return [minX, minY, maxX + 1, maxY + 1];
}
export function letterbox(bgr: any, mask: any, cv: any, size = FEATURE_SIZE): { bgrCanvas: any; maskCanvas: any } {
const scale = size / Math.max(bgr.rows, bgr.cols);
const nh = Math.max(1, Math.round(bgr.rows * scale));
const nw = Math.max(1, Math.round(bgr.cols * scale));
const resizedBgr = new cv.Mat();
const resizedMask = new cv.Mat();
const dsize = new cv.Size(nw, nh);
cv.resize(bgr, resizedBgr, dsize, 0, 0, cv.INTER_AREA);
cv.resize(mask, resizedMask, dsize, 0, 0, cv.INTER_AREA);
const bgrCanvas = cv.Mat.zeros(size, size, cv.CV_8UC3);
const maskCanvas = cv.Mat.zeros(size, size, cv.CV_8UC1);
const oy = Math.floor((size - nh) / 2);
const ox = Math.floor((size - nw) / 2);
const bgrRoi = bgrCanvas.roi(new cv.Rect(ox, oy, nw, nh));
resizedBgr.copyTo(bgrRoi);
bgrRoi.delete();
const maskRoi = maskCanvas.roi(new cv.Rect(ox, oy, nw, nh));
resizedMask.copyTo(maskRoi);
maskRoi.delete();
resizedBgr.delete();
resizedMask.delete();
return { bgrCanvas, maskCanvas };
}
export function hsHist(bgr: any, mask: any, cv: any): Float32Array {
const hsv = new cv.Mat();
cv.cvtColor(bgr, hsv, cv.COLOR_BGR2HSV);
const totalBins = HIST_H_BINS * HIST_S_BINS;
const hist = new Float32Array(totalBins);
const hsvData = hsv.data;
const maskData = mask.data;
for (let y = 0; y < bgr.rows; y++) {
const rowOffset = y * bgr.cols;
for (let x = 0; x < bgr.cols; x++) {
if (maskData[rowOffset + x] <= MASK_THRESHOLD) continue;
const idx = (rowOffset + x) * 3;
const h = hsvData[idx];
const s = hsvData[idx + 1];
if (s < 24) {
hist[0]++;
} else {
const hBin = Math.min(HIST_H_BINS - 1, Math.floor(h / 10));
const sBin = Math.min(HIST_S_BINS - 1, Math.floor(s / 16));
hist[hBin * HIST_S_BINS + sBin]++;
}
}
}
hsv.delete();
let sumSq = 0;
for (let i = 0; i < totalBins; i++) sumSq += hist[i] * hist[i];
const norm = Math.sqrt(sumSq) + 1e-6;
for (let i = 0; i < totalBins; i++) hist[i] /= norm;
return hist;
}
export function edgeMap(bgr: any, mask: any, cv: any): Float32Array {
const gray = new cv.Mat();
cv.cvtColor(bgr, gray, cv.COLOR_BGR2GRAY);
const gx = new cv.Mat();
const gy = new cv.Mat();
cv.Sobel(gray, gx, cv.CV_32F, 1, 0, 3);
cv.Sobel(gray, gy, cv.CV_32F, 0, 1, 3);
const mag = new cv.Mat();
cv.magnitude(gx, gy, mag);
const totalPixels = bgr.rows * bgr.cols;
const edge = new Float32Array(totalPixels);
const magData = mag.data32F;
const maskData = mask.data;
let sumSq = 0;
for (let i = 0; i < totalPixels; i++) {
if (maskData[i] > MASK_THRESHOLD) {
edge[i] = magData[i];
sumSq += magData[i] * magData[i];
}
}
const norm = Math.sqrt(sumSq) + 1e-6;
for (let i = 0; i < totalPixels; i++) edge[i] /= norm;
gray.delete();
gx.delete();
gy.delete();
mag.delete();
return edge;
}
export function estimateCardBg(cardBgr: any, cv: any): [number, number, number] {
const hsv = new cv.Mat();
cv.cvtColor(cardBgr, hsv, cv.COLOR_BGR2HSV);
const lowCard = new cv.Mat(cardBgr.rows, cardBgr.cols, cv.CV_8UC3, [165, 170, 60, 0]);
const highCard = new cv.Mat(cardBgr.rows, cardBgr.cols, cv.CV_8UC3, [179, 255, 185, 0]);
const mask = new cv.Mat();
cv.inRange(hsv, lowCard, highCard, mask);
const bList: number[] = [];
const gList: number[] = [];
const rList: number[] = [];
for (let y = 0; y < cardBgr.rows; y++) {
for (let x = 0; x < cardBgr.cols; x++) {
if (mask.ucharAt(y, x) === 0) continue;
bList.push(cardBgr.ucharAt(y, x * 3));
gList.push(cardBgr.ucharAt(y, x * 3 + 1));
rList.push(cardBgr.ucharAt(y, x * 3 + 2));
}
}
lowCard.delete();
highCard.delete();
mask.delete();
hsv.delete();
if (bList.length < 50) return [...FALLBACK_CARD_BGR];
const median = (values: number[]) => {
values.sort((a, b) => a - b);
return values[Math.floor(values.length / 2)];
};
return [median(bList), median(gList), median(rList)];
}
function opencvHue(b: number, g: number, r: number): { hue: number; saturation: number; value: number } {
const max = Math.max(b, g, r);
const min = Math.min(b, g, r);
const delta = max - min;
let hue = 0;
if (delta !== 0) {
if (max === r) hue = ((g - b) / delta) % 6;
else if (max === g) hue = (b - r) / delta + 2;
else hue = (r - g) / delta + 4;
hue *= 60;
if (hue < 0) hue += 360;
}
return { hue: hue / 2, saturation: max === 0 ? 0 : (delta / max) * 255, value: max };
}
/** 広い赤紫マスクに入っていても、カードより緑が強い画素は本体として残す。 */
function keepAsSprite(b: number, g: number, r: number, bg: readonly [number, number, number]): boolean {
const { saturation, value } = opencvHue(b, g, r);
const cardValue = Math.max(bg[0], bg[1], bg[2]);
if (g <= bg[1] + CARD_BG_GREEN_MARGIN && value <= cardValue + CARD_BG_VALUE_MARGIN) return false;
return saturation <= CARD_BG_MAX_SATURATION;
}
function largestComponent(binary: any, cv: any): any {
const labels = new cv.Mat();
const stats = new cv.Mat();
const centroids = new cv.Mat();
const numLabels = cv.connectedComponentsWithStats(binary, labels, stats, centroids, 8, cv.CV_32S);
if (numLabels <= 1) {
labels.delete();
stats.delete();
centroids.delete();
return binary.clone();
}
let maxArea = -1;
let keepLabel = 1;
for (let label = 1; label < numLabels; label++) {
const area = stats.intAt(label, cv.CC_STAT_AREA);
if (area > maxArea) {
maxArea = area;
keepLabel = label;
}
}
const minArea = Math.max(16, maxArea * 0.02);
const keepLabels = new Set<number>([keepLabel]);
for (let label = 1; label < numLabels; label++) {
if (label === keepLabel) continue;
const area = stats.intAt(label, cv.CC_STAT_AREA);
if (area < minArea) continue;
const x = stats.intAt(label, cv.CC_STAT_LEFT);
const y = stats.intAt(label, cv.CC_STAT_TOP);
const w = stats.intAt(label, cv.CC_STAT_WIDTH);
const h = stats.intAt(label, cv.CC_STAT_HEIGHT);
const touchesBorder = y <= 3 || (y + h) >= binary.rows - 3 || x <= 3 || (x + w) >= binary.cols - 3;
if (touchesBorder && (h <= 6 || w <= 6 || w / h > 3 || h / w > 3)) continue;
keepLabels.add(label);
}
const out = new cv.Mat(binary.rows, binary.cols, cv.CV_8UC1);
const labelsData = labels.data32S;
const pixels = binary.rows * binary.cols;
for (let i = 0; i < pixels; i++) out.data[i] = keepLabels.has(labelsData[i]) ? 255 : 0;
labels.delete();
stats.delete();
centroids.delete();
return out;
}
export function extractQuery(
cropBgr: any,
bgBgr: [number, number, number],
cv: any,
): { queryRgb: Float32Array; queryMask: Uint8Array } {
const hsv = new cv.Mat();
cv.cvtColor(cropBgr, hsv, cv.COLOR_BGR2HSV);
const lowBg = new cv.Mat(cropBgr.rows, cropBgr.cols, cv.CV_8UC3, [...SPRITE_BG_HSV_LO, 0]);
const highBg = new cv.Mat(cropBgr.rows, cropBgr.cols, cv.CV_8UC3, [...SPRITE_BG_HSV_HI, 0]);
const bgMask = new cv.Mat();
cv.inRange(hsv, lowBg, highBg, bgMask);
const [bgB, bgG, bgR] = bgBgr;
const sprite = new cv.Mat(cropBgr.rows, cropBgr.cols, cv.CV_8UC1);
const restored = new cv.Mat.zeros(cropBgr.rows, cropBgr.cols, cv.CV_8UC1);
for (let y = 0; y < cropBgr.rows; y++) {
for (let x = 0; x < cropBgr.cols; x++) {
const b = cropBgr.ucharAt(y, x * 3);
const g = cropBgr.ucharAt(y, x * 3 + 1);
const r = cropBgr.ucharAt(y, x * 3 + 2);
const dist = Math.hypot(b - bgB, g - bgG, r - bgR);
if (dist < BG_RGB_DISTANCE) bgMask.ucharPtr(y)[x] = 255;
const background = bgMask.ucharAt(y, x) > 0;
sprite.ucharPtr(y)[x] = background ? 0 : 255;
if (background && dist >= BG_RGB_DISTANCE && keepAsSprite(b, g, r, bgBgr)) restored.ucharPtr(y)[x] = 255;
}
}
let baseCount = 0;
for (let i = 0; i < sprite.data.length; i++) if (sprite.data[i] > 0) baseCount++;
if (baseCount / sprite.data.length < SPRITE_RESTORE_MAX_COVERAGE) {
const seed = new cv.Mat();
cv.bitwise_not(sprite, seed);
const distance = new cv.Mat();
cv.distanceTransform(seed, distance, cv.DIST_L2, 3);
const distanceData = distance.data32F;
const rightLimit = Math.floor(cropBgr.cols * SPRITE_RESTORE_RIGHT_LIMIT);
for (let y = 0; y < cropBgr.rows; y++) {
for (let x = 0; x < rightLimit; x++) {
const i = y * cropBgr.cols + x;
if (restored.data[i] > 0 && distanceData[i] <= SPRITE_RESTORE_RADIUS) sprite.data[i] = 255;
}
}
seed.delete();
distance.delete();
}
for (let b = 0; b < 2; b++) {
for (let y = 0; y < cropBgr.rows; y++) {
sprite.ucharPtr(y)[b] = 0;
sprite.ucharPtr(y)[cropBgr.cols - 1 - b] = 0;
}
for (let x = 0; x < cropBgr.cols; x++) {
sprite.ucharPtr(b)[x] = 0;
sprite.ucharPtr(cropBgr.rows - 1 - b)[x] = 0;
}
}
const kernel7 = cv.Mat.ones(7, 7, cv.CV_8U);
cv.morphologyEx(sprite, sprite, cv.MORPH_CLOSE, kernel7);
const largestSprite = largestComponent(sprite, cv);
const spriteBox = opaqueBbox(largestSprite.data, cropBgr.cols, cropBgr.rows, 40);
let canvases: { bgrCanvas: any; maskCanvas: any };
if (!spriteBox) {
const fullMask = new cv.Mat(cropBgr.rows, cropBgr.cols, cv.CV_8UC1, new cv.Scalar(255));
canvases = letterbox(cropBgr, fullMask, cv, FEATURE_SIZE);
fullMask.delete();
} else {
const [x0, y0, x1, y1] = spriteBox;
const rect = new cv.Rect(
Math.max(0, x0 - 2),
Math.max(0, y0 - 2),
Math.min(cropBgr.cols, x1 + 2) - Math.max(0, x0 - 2),
Math.min(cropBgr.rows, y1 + 2) - Math.max(0, y0 - 2),
);
const trimmedBgr = cropBgr.roi(rect);
const trimmedMask = largestSprite.roi(rect);
canvases = letterbox(trimmedBgr, trimmedMask, cv, FEATURE_SIZE);
trimmedBgr.delete();
trimmedMask.delete();
}
const queryRgb = new Float32Array(canvases.bgrCanvas.data);
const queryMask = new Uint8Array(canvases.maskCanvas.data);
hsv.delete();
lowBg.delete();
highBg.delete();
bgMask.delete();
restored.delete();
sprite.delete();
kernel7.delete();
largestSprite.delete();
canvases.bgrCanvas.delete();
canvases.maskCanvas.delete();
return { queryRgb, queryMask };
}
export function maskedNcc(queryRgb: Float32Array, queryMask: Uint8Array, templateRgb: Float32Array, templateMask: Uint8Array): number {
const pixels = FEATURE_SIZE * FEATURE_SIZE;
let count = 0;
let qSumB = 0;
let qSumG = 0;
let qSumR = 0;
let tSumB = 0;
let tSumG = 0;
let tSumR = 0;
for (let p = 0; p < pixels; p++) {
if (templateMask[p] <= MASK_THRESHOLD || queryMask[p] <= MASK_THRESHOLD) continue;
count++;
const qp = p * 3;
qSumB += queryRgb[qp];
qSumG += queryRgb[qp + 1];
qSumR += queryRgb[qp + 2];
tSumB += templateRgb[qp];
tSumG += templateRgb[qp + 1];
tSumR += templateRgb[qp + 2];
}
if (count < NCC_MIN_PIXELS) return -1;
const inv = 1 / count;
const qMeanB = qSumB * inv;
const qMeanG = qSumG * inv;
const qMeanR = qSumR * inv;
const tMeanB = tSumB * inv;
const tMeanG = tSumG * inv;
const tMeanR = tSumR * inv;
let dot = 0;
let qNormSq = 0;
let tNormSq = 0;
for (let p = 0; p < pixels; p++) {
if (templateMask[p] <= MASK_THRESHOLD || queryMask[p] <= MASK_THRESHOLD) continue;
const qp = p * 3;
const q0B = queryRgb[qp] - qMeanB;
const q0G = queryRgb[qp + 1] - qMeanG;
const q0R = queryRgb[qp + 2] - qMeanR;
const t0B = templateRgb[qp] - tMeanB;
const t0G = templateRgb[qp + 1] - tMeanG;
const t0R = templateRgb[qp + 2] - tMeanR;
dot += q0B * t0B + q0G * t0G + q0R * t0R;
qNormSq += q0B * q0B + q0G * q0G + q0R * q0R;
tNormSq += t0B * t0B + t0G * t0G + t0R * t0R;
}
return dot / (Math.sqrt(qNormSq) * Math.sqrt(tNormSq) + 1e-6);
}
function shiftedNcc(
qRgb: Float32Array,
qMask: Uint8Array,
tRgb: Float32Array,
tMask: Uint8Array,
dx: number,
dy: number,
): number {
const size = FEATURE_SIZE;
let count = 0;
let qSumB = 0;
let qSumG = 0;
let qSumR = 0;
let tSumB = 0;
let tSumG = 0;
let tSumR = 0;
const yStart = Math.max(0, dy);
const yEnd = Math.min(size, size + dy);
const xStart = Math.max(0, dx);
const xEnd = Math.min(size, size + dx);
for (let y = yStart; y < yEnd; y++) {
const qy = y - dy;
const tRow = y * size;
const qRow = qy * size;
for (let x = xStart; x < xEnd; x++) {
const qx = x - dx;
const tp = tRow + x;
const qp = qRow + qx;
if (tMask[tp] <= MASK_THRESHOLD || qMask[qp] <= MASK_THRESHOLD) continue;
count++;
const q3 = qp * 3;
const t3 = tp * 3;
qSumB += qRgb[q3];
qSumG += qRgb[q3 + 1];
qSumR += qRgb[q3 + 2];
tSumB += tRgb[t3];
tSumG += tRgb[t3 + 1];
tSumR += tRgb[t3 + 2];
}
}
if (count < NCC_MIN_PIXELS) return -1;
const inv = 1 / count;
const qMeanB = qSumB * inv;
const qMeanG = qSumG * inv;
const qMeanR = qSumR * inv;
const tMeanB = tSumB * inv;
const tMeanG = tSumG * inv;
const tMeanR = tSumR * inv;
let dot = 0;
let qNormSq = 0;
let tNormSq = 0;
for (let y = yStart; y < yEnd; y++) {
const qy = y - dy;
const tRow = y * size;
const qRow = qy * size;
for (let x = xStart; x < xEnd; x++) {
const qx = x - dx;
const tp = tRow + x;
const qp = qRow + qx;
if (tMask[tp] <= MASK_THRESHOLD || qMask[qp] <= MASK_THRESHOLD) continue;
const q3 = qp * 3;
const t3 = tp * 3;
const q0B = qRgb[q3] - qMeanB;
const q0G = qRgb[q3 + 1] - qMeanG;
const q0R = qRgb[q3 + 2] - qMeanR;
const t0B = tRgb[t3] - tMeanB;
const t0G = tRgb[t3 + 1] - tMeanG;
const t0R = tRgb[t3 + 2] - tMeanR;
dot += q0B * t0B + q0G * t0G + q0R * t0R;
qNormSq += q0B * q0B + q0G * q0G + q0R * q0R;
tNormSq += t0B * t0B + t0G * t0G + t0R * t0R;
}
}
return dot / (Math.sqrt(qNormSq) * Math.sqrt(tNormSq) + 1e-6);
}
const NCC_SEARCH_SHIFTS = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5];
/** ±5 では届かない位置ずれ。エースバーンは右へ 8px ずらすと一致する。 */
const NCC_OUTER_OFFSETS = [-12, -8, 8, 12];
function bestShiftedNcc(
queryRgb: Float32Array,
queryMask: Uint8Array,
templateRgb: Float32Array,
templateMask: Uint8Array,
dxValues: readonly number[],
dyValues: readonly number[],
best: number,
): number {
for (const dy of dyValues) {
for (const dx of dxValues) {
if (dx === 0 && dy === 0) continue;
const ncc = shiftedNcc(queryRgb, queryMask, templateRgb, templateMask, dx, dy);
if (ncc > best) {
best = ncc;
if (best >= 0.85) return best;
}
}
}
return best;
}
export function bestMaskedNcc(
queryRgb: Float32Array,
queryMask: Uint8Array,
templateRgb: Float32Array,
templateMask: Uint8Array,
histScore = 1,
): number {
let best = maskedNcc(queryRgb, queryMask, templateRgb, templateMask);
if (best >= 0.75) return best;
if (best < 0.15 && histScore < 0.35) return best;
best = bestShiftedNcc(queryRgb, queryMask, templateRgb, templateMask, NCC_SEARCH_SHIFTS, NCC_SEARCH_SHIFTS, best);
if (best >= 0.72) return best;
// 近い位置で形が合わない相手は、大きくずらした偶然の一致を採らない。
const outerX = bestShiftedNcc(queryRgb, queryMask, templateRgb, templateMask, NCC_OUTER_OFFSETS, NCC_SEARCH_SHIFTS, -1);
if (outerX >= 0.6) best = Math.max(best, outerX);
if (best >= 0.85) return best;
const outerY = bestShiftedNcc(queryRgb, queryMask, templateRgb, templateMask, NCC_SEARCH_SHIFTS, NCC_OUTER_OFFSETS, -1);
if (outerY >= 0.6) best = Math.max(best, outerY);
return best;
}
export function vectorDot(left: Float32Array, right: Float32Array): number {
let dot = 0;
for (let i = 0; i < left.length; i++) dot += left[i] * right[i];
return dot;
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/domain/score-match.ts
export type MatchFeatures = {
id: string;
ncc: number;
hist: number;
edge: number;
};
export type RankedMatch = {
id: string;
score: number;
ncc: number;
hist: number;
edge: number;
};
export type RankMatchesResult =
| { ok: true; matches: RankedMatch[] }
| { ok: false; code: "invalid_input" };
const SCORE_NCC_WEIGHT = 0.55;
const SCORE_HIST_WEIGHT = 0.3;
const SCORE_EDGE_WEIGHT = 0.15;
/** 3つの類似度を、重み 0.55 / 0.30 / 0.15 で1つのスコアにする。公開値は小数第3位。 */
export function matchScore(features: Pick<MatchFeatures, "ncc" | "hist" | "edge">): number {
const raw = SCORE_NCC_WEIGHT * features.ncc + SCORE_HIST_WEIGHT * features.hist + SCORE_EDGE_WEIGHT * features.edge;
if (raw <= 0) return 0;
const rounded = Math.round(raw * 1000) / 1000;
return rounded === 0 ? 0 : rounded;
}
/**
* 候補をスコアの降順に並べ、先頭 topk 件を返す。
* 同点は入力順を保つ。topk が 1 以上の整数でなければ invalid_input。
*/
export function rankMatches(candidates: readonly MatchFeatures[], topk: number): RankMatchesResult {
if (!Number.isInteger(topk) || topk < 1) {
return { ok: false, code: "invalid_input" };
}
const ranked = candidates.map((candidate, index) => ({
candidate,
index,
score: matchScore(candidate),
}));
ranked.sort((left, right) => {
if (left.score !== right.score) return right.score - left.score;
return left.index - right.index;
});
return {
ok: true,
matches: ranked.slice(0, Math.min(topk, ranked.length)).map(({ candidate, score }) => ({
id: candidate.id,
score,
ncc: candidate.ncc,
hist: candidate.hist,
edge: candidate.edge,
})),
};
}
pokemon-champions-api/src/module/pokemon-tools/recognizeOpponent/domain/confidence.ts
export type ConfidenceStatus = "ok" | "review";
export type ConfidenceWarning = "low_recognition_score" | "small_candidate_margin";
export type ConfidenceInput = {
score: number;
margin: number | null;
};
export type ConfidenceEvaluation = {
status: ConfidenceStatus;
warnings: ConfidenceWarning[];
};
const MIN_SCORE = 0.6;
const MIN_MARGIN = 0.05;
/** 1位のスコアと次点との差から、その枠をそのまま使ってよいかを返す。 */
export function evaluateConfidence(input: ConfidenceInput): ConfidenceEvaluation {
const warnings: ConfidenceWarning[] = [];
if (input.score < MIN_SCORE) warnings.push("low_recognition_score");
if (input.margin !== null && input.margin < MIN_MARGIN) warnings.push("small_candidate_margin");
return {
status: warnings.length === 0 ? "ok" : "review",
warnings,
};
}
④ ターミナルに結果を表示
サーバは認識した6体をJSON形式でSwiftに返却します。
Swift側では受信したJSONを解析し、ポケモンの名前・認識スコア・判定状態をターミナルに表示します。slot の昇順に1行ずつ出します。失敗したときは code を1行出します。
pokemon-champions-swift/Sources/CaptureCore/ServerMessage.swift
import Foundation
struct OpponentSlot: Decodable, Equatable, Sendable {
var slot: Int
var speciesName: String?
var displayName: String?
var score: Double?
var status: String?
var warnings: [String]?
}
enum ServerMessage: Equatable, Sendable {
case opponents([OpponentSlot])
case failure(code: String)
case unrecognized(String)
}
private struct FailureBody: Decodable {
var code: String
}
func parseServerMessage(_ text: String) -> ServerMessage {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard let data = trimmed.data(using: .utf8) else {
return .unrecognized(text)
}
let decoder = JSONDecoder()
if let slots = try? decoder.decode([OpponentSlot].self, from: data) {
return .opponents(slots)
}
if let failure = try? decoder.decode(FailureBody.self, from: data) {
return .failure(code: failure.code)
}
return .unrecognized(trimmed)
}
func describeImageResult(_ message: ServerMessage) -> String {
switch message {
case .opponents(let slots):
return slots
.sorted { $0.slot < $1.slot }
.map(describeSlot)
.joined(separator: "\n")
case .failure(let code):
return " \(code)"
case .unrecognized(let raw):
return " 応答を解釈できません: \(raw)"
}
}
private func describeSlot(_ slot: OpponentSlot) -> String {
let warnings = slot.warnings ?? []
let warningText = warnings.isEmpty ? "" : " [\(warnings.joined(separator: ", "))]"
let score = slot.score.map { String(format: "%g", $0) } ?? "-"
let status = slot.status ?? "-"
let name = slot.displayName ?? "(名前なし)"
let species = slot.speciesName ?? "null"
return " \(slot.slot). \(name) \(species) score=\(score) \(status)\(warningText)"
}
func describeServerMessage(_ message: ServerMessage) -> String {
switch message {
case .opponents(let slots):
let names = slots
.sorted { $0.slot < $1.slot }
.map { $0.displayName ?? "(名前なし)" }
if names.isEmpty {
return "相手: 0件"
}
return "相手: " + names.joined(separator: "、")
case .failure(let code):
return code
case .unrecognized(let raw):
return "応答を解釈できません: \(raw)"
}
}
成功したときの出力は、おおむね次の形です。先頭は保存と送信の記録、続く行が6体です。
保存しました /path/to/debug/iphone-mirroring/20261107-120000-000.png
ws://127.0.0.1:3000/capture へ iPhoneミラーリングを送ります
iPhoneミラーリング (123456 bytes)
1. ムクホーク staraptor score=0.8 ok
2. アシレーヌ primarina score=0.8 ok
3. コノヨザル annihilape score=0.8 review [small_candidate_margin]
4. ライチュウ raichu score=0.8 ok
5. オーロンゲ grimmsnarl score=0.8 ok
6. マスカーニャ meowscarada score=0.8 ok
図鑑に無い枠は、種族名の位置が null になり、警告に species_not_found が付きます。
1. イエッサン null score=0.764 review [species_not_found]
選出画面ではなかった画像は、枠の代わりに code が1行出ます。
selection_screen_not_found
これで、ゲーム画面から相手6体を自動取得するまでの処理がつながりました。
まとめ
今回は、過去に開発したiPhoneミラーリングの画面取得とOpenCVの画像認識を、WebSocketで接続しました。
これによって、選出画面から相手6体を特定し、ターミナルに表示できるようになりました。
次回は、認識した6体のポケモンをブラウザに表示する仕組みを作っていきます。




