wslでelixir その186

概要

wsl(wsl2じゃない)で、elixirやってみた。
Livebookで、Live2D + Speech + lipsync 書いてみた。

写真

setup

Mix.install([
  {:kino, "~> 0.19"}
])

サンプルコード

defmodule KinoHTML.Live2d do
  use Kino.JS
  use Kino.JS.Live
  def new(toolbox) do
    Kino.JS.Live.new(__MODULE__, toolbox)
  end
  def speak(kino, text) do
    Kino.JS.Live.cast(kino, {:speak, text})
  end
  @impl true
  def init(toolbox, ctx) do
    {:ok, assign(ctx, toolbox: toolbox)}
  end
  @impl true
  def handle_connect(ctx) do
    {:ok, ctx.assigns.toolbox, ctx}
  end
  @impl true
  def handle_cast({:speak, text}, ctx) do
    broadcast_event(ctx, "speak", %{text: text})
    {:noreply, ctx}
  end
  asset "main.js" do
    """
export async function init(ctx, toolbox) {
  await ctx.importJS("https://cubism.live2d.com/sdk-web/cubismcore/live2dcubismcore.min.js")
  await ctx.importJS("https://cdn.jsdelivr.net/gh/dylanNew/live2d/webgl/Live2D/lib/live2d.min.js")
  await ctx.importJS("https://cdn.jsdelivr.net/npm/pixi.js@6.5.2/dist/browser/pixi.min.js")
  await ctx.importJS("https://cdn.jsdelivr.net/npm/pixi-live2d-display/dist/index.min.js")
  const canvas = document.createElement("canvas");
  canvas.style.height = "480px";
  canvas.style.width = "600px";
  ctx.root.appendChild(canvas);
  const cubism2Model = "https://cdn.jsdelivr.net/gh/guansss/pixi-live2d-display/test/assets/shizuku/shizuku.model.json";
  await new Promise(resolve => setTimeout(resolve, 50));
  const app = new PIXI.Application({
    view: canvas,
    autoStart: true,
    backgroundAlpha: 0
  });
  const model2 = await PIXI.live2d.Live2DModel.from(cubism2Model);
  app.stage.addChild(model2);
  model2.scale.set(0.3);

  let isSpeaking = false;
  let nextMouthValue = 0.0;
  let lastUpdateTime = 0;
  if (model2.internalModel && model2.internalModel.motionManager) 
  {
    const motionManager = model2.internalModel.motionManager;
    const originalUpdate = motionManager.update;
    motionManager.update = function (coreModel, deltaTime) {
      const result = originalUpdate.call(this, coreModel, deltaTime);
      if (isSpeaking) 
      {
        const now = Date.now();
        if (now - lastUpdateTime > 60) 
        {
          nextMouthValue = Math.random() * 0.5;
          lastUpdateTime = now;
        }
        coreModel.setParamFloat("PARAM_MOUTH_OPEN_Y", nextMouthValue);
      } 
      else 
      {
        coreModel.setParamFloat("PARAM_MOUTH_OPEN_Y", 0.1);
      }
      return result;
    };
  }
  
  ctx.handleEvent("speak", ({ text }) => {
    if (!text) return;
    const utterance = new SpeechSynthesisUtterance(text);
    utterance.onstart = () => {
      isSpeaking = true;
      lastUpdateTime = Date.now();
    };
    const stopLipSync = () => {
      isSpeaking = false;
    };
    utterance.onend = stopLipSync;
    utterance.onerror = stopLipSync;
    window.speechSynthesis.cancel();
    window.speechSynthesis.speak(utterance);
  });
}
    """
  end
end




kino = KinoHTML.Live2d.new(0)
   
KinoHTML.Live2d.speak(kino, "画面が表示されましたか？今度はうまくいくはずです。")

以上。