LoginSignup
4
1

More than 1 year has passed since last update.

@poruruba

手のジェスチャーでショートカットキー入力

Last updated at Posted at 2022-03-01

M5Stack向けに、ジェスチャー検知するユニットがあります。

上、下、右、左、前、後、時計回り、反時計回り、手を振る、といった計9つのジェスチャーがプリセットされています。
今回はこれを、M5StickCに接続し、BLEでHIDとして接続してふるまうようにします。
例えば、Windowsにキーボードとして登録し、手を使ったジェスチャーに反応して、ショートカットキーを入力します。そうすることで、パワーポイントのページを進めたり、Zoomでミュートさせたりします。

ソースコードもろもろは以下のGitHubに上げておきました。

poruruba/GestureHid

※ただし、ジェスチャーは結構な頻度で誤検出するので、作ってみて思ったのですが、あまり実用的ではないような気がします。。。

M5StickCをBLEのHIDとして動作させる

ソースコードとしては、こんな感じです。

src/main.cpp
#include <M5StickC.h>
#include <BLEDevice.h>
#include <BLEUtils.h>
#include <BLEServer.h>
#include <BLE2902.h>
#include <BLEHIDDevice.h>
#include <HIDTypes.h>

class MyCallbacks : public BLEServerCallbacks {
  void onConnect(BLEServer* pServer){
    ble_connected = true;
    digitalWrite(LED_PORT, LOW);
    BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
    desc->setNotifications(true);
  }

  void onDisconnect(BLEServer* pServer){
    ble_connected = false;
    digitalWrite(LED_PORT, HIGH);
    BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
    desc->setNotifications(false);
  }
};

void taskServer(void*){
  BLEDevice::init(deviceName);

  BLEServer *pServer = BLEDevice::createServer();
  pServer->setCallbacks(new MyCallbacks());

  hid = new BLEHIDDevice(pServer);
  input = hid->inputReport(1); // <-- input REPORTID from report map
  output = hid->outputReport(1); // <-- output REPORTID from report map

  hid->manufacturer()->setValue(manufacturerName);

  hid->pnp(0x02, 0xe502, 0xa111, 0x0210);
  hid->hidInfo(0x00,0x02);

    const uint8_t report[] = {
      USAGE_PAGE(1),      0x01,       // Generic Desktop Ctrls
      USAGE(1),           0x06,       // Keyboard
      COLLECTION(1),      0x01,       // Application
      REPORT_ID(1),       0x01,        //   Report ID (1)
      USAGE_PAGE(1),      0x07,       //   Kbrd/Keypad
      USAGE_MINIMUM(1),   0xE0,
      USAGE_MAXIMUM(1),   0xE7,
      LOGICAL_MINIMUM(1), 0x00,
      LOGICAL_MAXIMUM(1), 0x01,
      REPORT_SIZE(1),     0x01,       //   1 byte (Modifier)
      REPORT_COUNT(1),    0x08,
      HIDINPUT(1),           0x02,       //   Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position
      REPORT_COUNT(1),    0x01,       //   1 byte (Reserved)
      REPORT_SIZE(1),     0x08,
      HIDINPUT(1),           0x01,       //   Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position
      REPORT_COUNT(1),    0x06,       //   6 bytes (Keys)
      REPORT_SIZE(1),     0x08,
      LOGICAL_MINIMUM(1), 0x00,
      LOGICAL_MAXIMUM(1), 0x65,       //   101 keys
      USAGE_MINIMUM(1),   0x00,
      USAGE_MAXIMUM(1),   0x65,
      HIDINPUT(1),           0x00,       //   Data,Array,Abs,No Wrap,Linear,Preferred State,No Null Position
      REPORT_COUNT(1),    0x05,       //   5 bits (Num lock, Caps lock, Scroll lock, Compose, Kana)
      REPORT_SIZE(1),     0x01,
      USAGE_PAGE(1),      0x08,       //   LEDs
      USAGE_MINIMUM(1),   0x01,       //   Num Lock
      USAGE_MAXIMUM(1),   0x05,       //   Kana
      HIDOUTPUT(1),          0x02,       //   Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position,Non-volatile
      REPORT_COUNT(1),    0x01,       //   3 bits (Padding)
      REPORT_SIZE(1),     0x03,
      HIDOUTPUT(1),          0x01,       //   Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position,Non-volatile
      END_COLLECTION(0)
    };

  hid->reportMap((uint8_t*)report, sizeof(report));
  hid->startServices();

  BLEAdvertising *pAdvertising = pServer->getAdvertising();
  pAdvertising->setAppearance(HID_KEYBOARD);
  pAdvertising->addServiceUUID(hid->hidService()->getUUID());
  pAdvertising->start();
  hid->setBatteryLevel(7);

  Serial.println("Advertising started!");
  delay(portMAX_DELAY);
};

キー入力を発生させる

WinキーやCtrlキー、Altキー、Shiftキーといった装飾キーといくつかのキー入力の組み合わせでの入力の場合と、任意の文字列入力の場合の2種類を用意しました。

src/main.cpp
void sendKeys(uint8_t mod, const uint8_t *keys, uint8_t num_key){
  BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
  if( !desc->getNotifications() )
    return;

  uint8_t msg[] = {mod, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
  for( int i = 0 ; i < num_key && i < 6 ; i++ ){
    msg[2 + i] = keys[i];
  }

  input->setValue(msg, sizeof(msg));
  input->notify();

  uint8_t msg1[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
  input->setValue(msg1, sizeof(msg1));
  input->notify();

  delay(20);
}

void sendKeyString(const char* ptr){
  BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
  if( !desc->getNotifications() )
    return;

  // 1文字ずつHID(BLE)で送信
  while(*ptr){
    if( *ptr >= ASCIIMAP_SIZE_JP ){
      ptr++;
      continue;
    }
    KEYMAP map = asciimap_jp[(uint8_t)*ptr];
    uint8_t msg[] = {map.modifier, 0x00, map.usage, 0x00, 0x00, 0x00, 0x00, 0x00};
    input->setValue(msg, sizeof(msg));
    input->notify();
    ptr++;

    uint8_t msg1[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
    input->setValue(msg1, sizeof(msg1));
    input->notify();

    delay(20);
  }
}

ジェスチャーユニットの制御

platformio.iniを示します。このうち「DFRobot_PAJ7620U2」がジェスチャーユニットの制御用のドライバです。

platformio.ini
[env:m5stick-c]
platform = espressif32
board = m5stick-c
framework = arduino
monitor_speed = 115200
upload_port = COM4
monitor_port = COM4
lib_deps = 
	dfrobot/DFRobot_PAJ7620U2@^1.0.1
	m5stack/M5StickC@^0.2.5

使い方です。

src/main.cpp
#include <DFRobot_PAJ7620U2.h>
DFRobot_PAJ7620U2 paj;

void setup(){
・・・
  while(paj.begin() != 0){
    Serial.println("initial PAJ7620U2 failure!");
    delay(500);
  }
  Serial.println("PAJ7620U2 initialized");
・・・
}

void loop(){
・・・
    DFRobot_PAJ7620U2::eGesture_t gesture = paj.getGesture();
    if(gesture != paj.eGestureNone ){
    /* 
      * "None","Right","Left", "Up", "Down", "Forward", "Backward", "Clockwise", "Anti-Clockwise", "Wave",
      * "WaveSlowlyDisorder", "WaveSlowlyLeftRight", "WaveSlowlyUpDown", "WaveSlowlyForwardBackward"
    */
      String description  = paj.gestureDescription(gesture);
      Serial.println("Gesture = " + description);
・・・
   }
・・・
}

キーマクロの定義

以下、例です。

src/main.cpp
const KEYMACRO desktop_macros[] = {
  {
    ASIGN_Left,
    "デスクトップ切替(←)",
    TYPE_KEYMAP,
    .keymap = {
      0x50, KEY_MASK_WIN | KEY_MASK_CTRL
    },
  },
  {
    ASIGN_Right,
    "デスクトップ切替(→)",
    TYPE_KEYMAP,
    .keymap = {
      0x4f, KEY_MASK_WIN | KEY_MASK_CTRL
    },
  },
  {
    ASIGN_BtnA,
    "デスクトップ表示",
    TYPE_KEYMAP,
    .keymap = {
      asciimap_jp['d'].usage, KEY_MASK_WIN
    }
  },
  MACRO_END
};

上記のようなキーマクロのセットは、OSの状態や稼働中のアプリケーションによって切り替えたいところかと思いますので、M5StickCの側面のボタンで切り替えられるようにしました。
先ほどのセットは、通常のデスクトップ操作用ですが、それ以外にもPowerPoint用、Zoom用も用意しました。
都合のよいように、適宜追加、変更をしてみてください。

ソースコード全体

最後に、ソースコード全体を示しておきます。

src/main.cpp
#include <M5StickC.h>
#include <BLEDevice.h>
#include <BLEUtils.h>
#include <BLEServer.h>
#include <BLE2902.h>
#include <BLEHIDDevice.h>
#include <HIDTypes.h>
#include <DFRobot_PAJ7620U2.h>
#include "asciimap.h"

#define LED_PORT  GPIO_NUM_10
const char* deviceName = "GestureHid";
const char* manufacturerName = "M5StickC";

enum MACRO_ASIGN{
  ASIGN_None = 0,
  ASIGN_Right,
  ASIGN_Left,
  ASIGN_Up,
  ASIGN_Down,
  ASIGN_Forward,
  ASIGN_Backward,
  ASIGN_Clockwise,
  ASIGN_Anti_Clockwise,
  ASIGN_Wave,
  ASIGN_BtnA,
  ASIGN_BtnB,
//  ASIGN_BtnC,
};

enum MACRO_TYPE {
  TYPE_TEXT,
  TYPE_KEYMAP,
  TYPE_KEYSET
};

typedef struct{
  MACRO_ASIGN asign;
  const char *title;
  MACRO_TYPE type;
//  union{
    KEYMAP keymap;
    struct{
      uint8_t num_key;
      uint8_t keys[6];
      uint8_t mod;
    } keyset;
    struct{
      const char *text;
    } text;
//  };
} KEYMACRO;
#define MACRO_END { ASIGN_None }

typedef struct{
  const char *title;
  const KEYMACRO *macros;
}MACROPANEL;

const KEYMACRO desktop_macros[] = {
  {
    ASIGN_Left,
    "デスクトップ切替(←)",
    TYPE_KEYMAP,
    .keymap = {
      0x50, KEY_MASK_WIN | KEY_MASK_CTRL
    },
  },
  {
    ASIGN_Right,
    "デスクトップ切替(→)",
    TYPE_KEYMAP,
    .keymap = {
      0x4f, KEY_MASK_WIN | KEY_MASK_CTRL
    },
  },
  {
    ASIGN_BtnA,
    "デスクトップ表示",
    TYPE_KEYMAP,
    .keymap = {
      asciimap_jp['d'].usage, KEY_MASK_WIN
    }
  },
  MACRO_END
};
const KEYMACRO powerpoint_macros[] = {
  {
    ASIGN_Left,
    "前ページ",
    TYPE_KEYMAP,
    .keymap = {
      0x4b, 0 
    },
  },
  {
    ASIGN_Right,
    "次ページ",
    TYPE_KEYMAP,
    .keymap = {
      0x4e, 0 
    },
  },
  {
    ASIGN_BtnA,
    "スライドシー開始",
    TYPE_KEYMAP,
    .keymap = {
      0x3e, 0 
    },
  },
  {
    ASIGN_BtnB,
    "スライドショー終了",
    TYPE_KEYMAP,
    .keymap = {
      0x29, 0
    },
  },
  MACRO_END
};
const KEYMACRO zoom_macros[] = {
  {
    ASIGN_Wave,
    "ミュート",
    TYPE_KEYMAP,
    .keymap = {
      asciimap_jp['a'].usage, KEY_MASK_ALT
    },
  },
  {
    ASIGN_BtnA,
    "手を挙げる",
    TYPE_KEYMAP,
    .keymap = {
      asciimap_jp['y'].usage, KEY_MASK_ALT
    },
  },
  MACRO_END
};
const KEYMACRO none_macros[] = {
  MACRO_END
};

#define NUM_OF_PANEL  4
const MACROPANEL macropanels[NUM_OF_PANEL] = {
  {
    "Desktop",
    desktop_macros
  },
  {
    "PowerPoint",
    powerpoint_macros
  },
  {
    "Zoom",
    zoom_macros
  },
  {
    "None",
    none_macros
  }
};

DFRobot_PAJ7620U2 paj;
BLEHIDDevice* hid;
BLECharacteristic* input;
BLECharacteristic* output;

unsigned char current_panel = 0;
bool ble_connected = false;

KEYMACRO findKeyMacro(uint8_t current_panel, enum MACRO_ASIGN asign);
void sendMacroKey(KEYMACRO macro);
void sendKeys(uint8_t mod, const uint8_t *keys, uint8_t num_key);
void sendKeyString(const char* ptr);

class MyCallbacks : public BLEServerCallbacks {
  void onConnect(BLEServer* pServer){
    ble_connected = true;
    digitalWrite(LED_PORT, LOW);
    BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
    desc->setNotifications(true);
  }

  void onDisconnect(BLEServer* pServer){
    ble_connected = false;
    digitalWrite(LED_PORT, HIGH);
    BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
    desc->setNotifications(false);
  }
};

void taskServer(void*){
  BLEDevice::init(deviceName);

  BLEServer *pServer = BLEDevice::createServer();
  pServer->setCallbacks(new MyCallbacks());

  hid = new BLEHIDDevice(pServer);
  input = hid->inputReport(1); // <-- input REPORTID from report map
  output = hid->outputReport(1); // <-- output REPORTID from report map

  hid->manufacturer()->setValue(manufacturerName);

  hid->pnp(0x02, 0xe502, 0xa111, 0x0210);
  hid->hidInfo(0x00,0x02);

    const uint8_t report[] = {
      USAGE_PAGE(1),      0x01,       // Generic Desktop Ctrls
      USAGE(1),           0x06,       // Keyboard
      COLLECTION(1),      0x01,       // Application
      REPORT_ID(1),       0x01,        //   Report ID (1)
      USAGE_PAGE(1),      0x07,       //   Kbrd/Keypad
      USAGE_MINIMUM(1),   0xE0,
      USAGE_MAXIMUM(1),   0xE7,
      LOGICAL_MINIMUM(1), 0x00,
      LOGICAL_MAXIMUM(1), 0x01,
      REPORT_SIZE(1),     0x01,       //   1 byte (Modifier)
      REPORT_COUNT(1),    0x08,
      HIDINPUT(1),           0x02,       //   Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position
      REPORT_COUNT(1),    0x01,       //   1 byte (Reserved)
      REPORT_SIZE(1),     0x08,
      HIDINPUT(1),           0x01,       //   Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position
      REPORT_COUNT(1),    0x06,       //   6 bytes (Keys)
      REPORT_SIZE(1),     0x08,
      LOGICAL_MINIMUM(1), 0x00,
      LOGICAL_MAXIMUM(1), 0x65,       //   101 keys
      USAGE_MINIMUM(1),   0x00,
      USAGE_MAXIMUM(1),   0x65,
      HIDINPUT(1),           0x00,       //   Data,Array,Abs,No Wrap,Linear,Preferred State,No Null Position
      REPORT_COUNT(1),    0x05,       //   5 bits (Num lock, Caps lock, Scroll lock, Compose, Kana)
      REPORT_SIZE(1),     0x01,
      USAGE_PAGE(1),      0x08,       //   LEDs
      USAGE_MINIMUM(1),   0x01,       //   Num Lock
      USAGE_MAXIMUM(1),   0x05,       //   Kana
      HIDOUTPUT(1),          0x02,       //   Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position,Non-volatile
      REPORT_COUNT(1),    0x01,       //   3 bits (Padding)
      REPORT_SIZE(1),     0x03,
      HIDOUTPUT(1),          0x01,       //   Const,Array,Abs,No Wrap,Linear,Preferred State,No Null Position,Non-volatile
      END_COLLECTION(0)
    };

  hid->reportMap((uint8_t*)report, sizeof(report));
  hid->startServices();

  BLEAdvertising *pAdvertising = pServer->getAdvertising();
  pAdvertising->setAppearance(HID_KEYBOARD);
  pAdvertising->addServiceUUID(hid->hidService()->getUUID());
  pAdvertising->start();
  hid->setBatteryLevel(7);

  Serial.println("Advertising started!");
  delay(portMAX_DELAY);
};

void setup()
{
  M5.begin(true, true, true);
  Serial.println("setup start");

  M5.Axp.ScreenBreath(9);
  M5.Lcd.setRotation(0);
  M5.Lcd.setTextSize(2);

  M5.Lcd.fillScreen(TFT_BLACK);
  M5.Lcd.setCursor(0, 0);
  M5.Lcd.print("Starting...");

  pinMode(LED_PORT, OUTPUT);
  digitalWrite(LED_PORT, HIGH);

  while(paj.begin() != 0){
    Serial.println("initial PAJ7620U2 failure!");
    delay(500);
  }
  Serial.println("PAJ7620U2 initialized");

  M5.Lcd.fillScreen(TFT_BLACK);
  M5.Lcd.setCursor(0, 0);
  M5.Lcd.print(macropanels[current_panel].title);

  xTaskCreate(taskServer, "server", 20000, NULL, 5, NULL);

  Serial.println("setup end");
}

void loop()
{
  M5.update();

  if( M5.Axp.GetBtnPress() == 2 ){
    current_panel++;
    if( current_panel >= NUM_OF_PANEL )
      current_panel = 0;

    M5.Lcd.fillScreen(TFT_BLACK);
    M5.Lcd.setCursor(0, 0);
    M5.Lcd.print(macropanels[current_panel].title);

    return;
  }

  if( ble_connected ){
    DFRobot_PAJ7620U2::eGesture_t gesture = paj.getGesture();
    if(gesture != paj.eGestureNone ){
    /* 
      * "None","Right","Left", "Up", "Down", "Forward", "Backward", "Clockwise", "Anti-Clockwise", "Wave",
      * "WaveSlowlyDisorder", "WaveSlowlyLeftRight", "WaveSlowlyUpDown", "WaveSlowlyForwardBackward"
    */
      String description  = paj.gestureDescription(gesture);
      Serial.println("Gesture = " + description);

      KEYMACRO macro = { ASIGN_None };
      if( gesture == paj.eGestureRight ){
        macro = findKeyMacro(current_panel, ASIGN_Right );
      }else
      if( gesture == paj.eGestureLeft ){
        macro = findKeyMacro(current_panel, ASIGN_Left );
      }else
      if( gesture == paj.eGestureUp ){
        macro = findKeyMacro(current_panel, ASIGN_Up );
      }else
      if( gesture == paj.eGestureDown ){
        macro = findKeyMacro(current_panel, ASIGN_Down );
      }else
      if( gesture == paj.eGestureForward ){
        macro = findKeyMacro(current_panel, ASIGN_Forward );
      }else
      if( gesture == paj.eGestureBackward ){
        macro = findKeyMacro(current_panel, ASIGN_Backward );
      }else
      if( gesture == paj.eGestureClockwise ){
        macro = findKeyMacro(current_panel, ASIGN_Clockwise );
      }else
      if( gesture == paj.eGestureAntiClockwise ){
        macro = findKeyMacro(current_panel, ASIGN_Anti_Clockwise );
      }else
      if( gesture == paj.eGestureWave ){
        macro = findKeyMacro(current_panel, ASIGN_Wave );
      }

      sendMacroKey(macro);
      delay(100);
    }

    if( M5.BtnA.wasPressed() ){
      Serial.println("M5.BtnA wasPressed");
      KEYMACRO macro = findKeyMacro(current_panel, ASIGN_BtnA );
      sendMacroKey(macro);
      delay(100);
    }
    if( M5.BtnB.wasPressed() ){
      Serial.println("M5.BtnB wasPressed");
      KEYMACRO macro = findKeyMacro(current_panel, ASIGN_BtnB );
      sendMacroKey(macro);
      delay(100);
    }
  }

  delay(1);
}

void sendMacroKey(KEYMACRO macro){
  if( macro.asign == ASIGN_None )
    return;

  if( macro.type == TYPE_TEXT ){
    sendKeyString(macro.text.text);
  }else
  if( macro.type == TYPE_KEYMAP ){
    sendKeys(macro.keymap.modifier, &macro.keymap.usage, 1);
  }else
  if( macro.type == TYPE_KEYSET ){
    sendKeys(macro.keyset.mod, macro.keyset.keys, macro.keyset.num_key);
  }
}

void sendKeys(uint8_t mod, const uint8_t *keys, uint8_t num_key){
  BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
  if( !desc->getNotifications() )
    return;

  uint8_t msg[] = {mod, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
  for( int i = 0 ; i < num_key && i < 6 ; i++ ){
    msg[2 + i] = keys[i];
  }

  input->setValue(msg, sizeof(msg));
  input->notify();

  uint8_t msg1[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
  input->setValue(msg1, sizeof(msg1));
  input->notify();

  delay(20);
}

void sendKeyString(const char* ptr){
  BLE2902* desc = (BLE2902*)input->getDescriptorByUUID(BLEUUID((uint16_t)0x2902));
  if( !desc->getNotifications() )
    return;

  // 1文字ずつHID(BLE)で送信
  while(*ptr){
    if( *ptr >= ASCIIMAP_SIZE_JP ){
      ptr++;
      continue;
    }
    KEYMAP map = asciimap_jp[(uint8_t)*ptr];
    uint8_t msg[] = {map.modifier, 0x00, map.usage, 0x00, 0x00, 0x00, 0x00, 0x00};
    input->setValue(msg, sizeof(msg));
    input->notify();
    ptr++;

    uint8_t msg1[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
    input->setValue(msg1, sizeof(msg1));
    input->notify();

    delay(20);
  }
}

KEYMACRO findKeyMacro(uint8_t current_panel, enum MACRO_ASIGN asign){
  int i = 0;
  while(macropanels[current_panel].macros[i].asign != ASIGN_None ){
    if( macropanels[current_panel].macros[i].asign == asign )
      return macropanels[current_panel].macros[i];
    i++;
  }
  
  return { ASIGN_None };
}

参考

M5StickCのBLE HID化は以下が参考になります。
　ESP32でキーボードショートカットを作ってしまおう

以上

4
1
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
4
1