Video + A.I. real-time classifier in the browser (ml5.js)

Last updated at 2019-05-17Posted at 2019-05-17

##はじめに

こんにちは streampack チームのメディです。
https://cloudpack.jp/service/option/streampack.html

##Copyrights
Sintel
© copyright Blender Foundation | www.sintel.org

Street in India
copyleft video | https://pixabay.com/videos/india-people-asia-street-asian-1643/

##What is ml5.js ・ ml5.jsとは
From ml5.js homepage

The library provides access to machine learning algorithms and models in the browser, building on top of TensorFlow.js with no other external dependencies.

ml5.jsのホームページから

このライブラリは、TensorFlow.jsの上に構築されたブラウザ内の機械学習アルゴリズムおよびモデルへのアクセスを提供し、他の外部の依存関係はありません。

##Objective・目的
Learning how to get real time classification on the browser side from a video stream .

ブラウザ側でビデオストリームからリアルタイム分類を取得する方法を学ぶ。

##Tools・ツール

This example is using the MobileNet model developed by Google.
この例では、Googleが開発したMobileNetモデルを使用しています。

For playing the video stream, we use videojs. Videojs is an open sources media player for the web.
ビデオストリームを再生するには、videojsを使用します。 Videojsは、Web用のオープンソースのメディアプレーヤーです。

##Implementation ・実装

For every frame, if there are detections, there is a callback with the following information :

Object type
Detection confidence (between 0 and 1)

すべてのフレームについて、検出があると、次の情報を含むコールバックがあります。

オブジェクトタイプ
検出の信頼性（0から1の間）

In this demo, you can choose the video stream source and you can adjust the detection threshold.

Multiple video stream type are supported (HLS, MPEG-Dash, mp4, webm).

このデモでは、ビデオストリームのソースを選択したり、検出しきい値を調整したりできます。

マルチビデオストリームタイプがサポートされています（HLS、MPEG-Dash、mp4、webm）。

<!DOCTYPE html>
<html lang="en">

<head>
  <meta charset="UTF-8">

  <title>HLS/MPEG-Dash/MP4/Webm + AI Classifier</title>
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta http-equiv="X-UA-Compatible" content="ie=edge">

  <!-- Bootstrap CSS -->
  <link rel="stylesheet" href="//stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" crossorigin="anonymous" playsinline>

  <!-- jQuery & Bootstrap JS -->
  <script src="//code.jquery.com/jquery-3.3.1.slim.min.js" crossorigin="anonymous"></script>
  <script src="//cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" crossorigin="anonymous"></script>
  <script src="//stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" crossorigin="anonymous"></script>

  <!-- Videojs player -->
  <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/video.js/7.4.1/video-js.min.css" />
  <script src="//cdnjs.cloudflare.com/ajax/libs/video.js/7.4.1/video.min.js"></script>
  <!-- Videojs dash support -->
  <script src="//cdnjs.cloudflare.com/ajax/libs/dashjs/2.9.3/dash.all.min.js"></script>
  <script src="//cdnjs.cloudflare.com/ajax/libs/videojs-contrib-dash/2.11.0/videojs-dash.min.js"></script>

  <!-- p5.jsramework & ml5 library -->
  <script src="//cdnjs.cloudflare.com/ajax/libs/p5.js/0.8.0/p5.min.js" ></script>
  <script src="//cdnjs.cloudflare.com/ajax/libs/p5.js/0.8.0/addons/p5.dom.min.js"  ></script>
  <script src="//unpkg.com/ml5@0.2.3/dist/ml5.min.js"></script>


</head>

<body>
  <div style="margin:30px;">
  <h4>ml5.js + MobileNet </h4>
  <h4>Real time video classification</h4>
  <i id="status">Loading MobileNet Model...</i>
  
  <div class="form-group row">
    <h3 for="inputUrl" class="col-sm-12 col-form-label">Video stream URL (HLS, MPEG-Dash, mp4 or Webm) :</h3>
    <div class="col-sm-10">
      <input type="url" class="form-control" id="inputUrl" value="https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8" placeholder="Stream URL">
    </div>
    <div>
        <button type="button" class="btn btn-primary" onclick="handleUrl()">Reload Player</button>
    </div>
  
  </div>
  <div>
    <label>Detection Threshold: </label>
    <input id="threshold" type="number" value="40" min="0" max="100" step="10"/>
    <label>%<label/>
  </div>
  

  <video class="video-js vjs-default-skin" id="video" controls preload="auto" muted autoplay playsinline >
    <source src="https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8" type="application/x-mpegURL" width="1280" height="720" >
  </video>

  <table class="table">
    <thead class="thead-light">
      <tr>
        <th scope="col" style="width: 10%">#</th>
        <th scope="col" style="width: 30%">Label</th>
        <th scope="col" style="width: 10%">Confidence in percent </th>
      </tr>
    </thead>
    <tbody>
      <tr>
        <th scope="row">1</th>
        <td id="0_l"></td>
        <td id="0_c"></td>
      </tr>
      <tr>
        <th scope="row">2</th>
        <td id="1_l"></td>
        <td id="1_c"></td>
      </tr>
      <tr>
        <th scope="row">3</th>
        <td id="2_l"></td>
        <td id="2_c"></td>
      </tr>
    </tbody>
  </table>

</div>
  <script>
    var video = document.getElementById('video');
    let classifier;
    var player = videojs('video', {
      fluid: true
    });

    window.player = player;
    video.width = 1280;
    video.height = 720;
    video.crossOrigin = "anonymous";
    
    // Clearing the detection table every 3 sec
    var intervalID = window.setInterval(clearTable, 3000);


    function setup() {
      noCanvas();
      // Create a camera input
      // Initialize the Image Classifier method with MobileNet and the video as the second argument
      classifier = ml5.imageClassifier('MobileNet', video, modelReady);
    }

    function modelReady() {
      // Change the status of the model once its ready
      select('#status').html('Model Loaded');
      // Call the classifyVideo function to start classifying the video
      classifyVideo();
    }

    // Get a prediction for the current video frame
    function classifyVideo() {
      classifier.predict(gotResult);
    }

    // When we get a result
    function gotResult(err, results) {
      // The results are in an array ordered by probability.

      if (results!=null) {
        for (var i = 0; i < results.length; i++) {
          var confidence = 100 * nf(results[i].confidence, 0, 2);
          
          if ( confidence >= getThreshold()){
            select(`#${i}_l`).html(results[i].label);
            select(`#${i}_c`).html(confidence);
          }
        }
      }
      
      classifyVideo();
    }
    function handleUrl(){
      // Relaoding player & update src
      var streamUrl =  select('#inputUrl').value();
      var streamType = "";
      if (streamUrl.match(/m3u8/)){
        streamType  = "application/x-mpegURL";
      }
      if (streamUrl.match(/mpd/)){
        streamType  = "application/dash+xml";
      }
      if (streamUrl.match(/mp4/)){
        streamType  = "video/mp4";
      }
      if (streamUrl.match(/webm/)){
        streamType  = "video/webm";
      }    
      player.src(
        [
          { type: streamType, src: streamUrl }
        ]
      );
    }
    
    function getThreshold(){
      return   select('#threshold').value(); 
    }
    
    function clearTable(){
      for (var i =0;i<3;i++){
        select(`#${i}_l`).html("");
        select(`#${i}_c`).html("");
      }
    }
  </script>
</body>

</html>

Compatibility ・ウェブブラウザの互換性

Desktop browsers only for now.
デスクトップブラウザのみ。

Browser	Compatibility
Desktop Safari 12	yes
Desktop Chrome 72	yes
Desktop Firefox 66	yes

##Information sources ・情報元

https://ml5js.org/
https://github.com/ml5js/ml5-library
https://www.youtube.com/watch?v=yNkAuWz5lnY
https://github.com/videojs
https://pixabay.com/videos/india-people-asia-street-asian-1643/

You get articles that match your needs
You can efficiently read back useful information
You can use dark theme

What you can do with signing up