Real-Time Object Detection on Colab

Introduction

The operation was tested under the following conditions:

・Environment: Google Colaboratory
・Date: November 19, 2025
・Runtime: T4 GPU

1. Preparation

Mount Google Drive

from google.colab import drive
drive.mount('/content/drive')

Move to the directory where files will be saved

# Modify according to your environment
%cd /content/drive/MyDrive/RealTimePrediction/

Import required libraries

import IPython
from google.colab import output
import cv2
import numpy as np
from PIL import Image
from io import BytesIO
import base64

2. Check camera operation

Sample program for edge detection

def run(img_str):
    #decode to image
    decimg = base64.b64decode(img_str.split(',')[1], validate=True)
    decimg = Image.open(BytesIO(decimg))
    decimg = np.array(decimg, dtype=np.uint8);
    decimg = cv2.cvtColor(decimg, cv2.COLOR_BGR2RGB)

    gray = cv2.cvtColor(decimg, cv2.COLOR_BGR2GRAY)

    #############your process###############
    #ret, img2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)
    #out_img = cv2.Canny(decimg,100,200)
    height, width = gray.shape

    #img2 = cv2.resize(img2, (width//2, height//2 ))
    img2 = cv2.Canny( gray, 50, 51 )
    img2 = cv2.resize(img2, (width//2, height//2 ))

    out_img = img2

    #############your process###############
    #encode to string
    _, encimg = cv2.imencode(".jpg", out_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
    #img_str = encimg.tostring()
    img_str = encimg.tobytes()
    img_str = "data:image/jpeg;base64," + base64.b64encode(img_str).decode('utf-8')
    return IPython.display.JSON({'img_str': img_str})
    
output.register_callback('notebook.run', run)

# Function for camera access

from IPython.display import display, Javascript
from google.colab.output import eval_js

def use_cam(quality=0.8):
  js = Javascript('''
    async function useCam(quality) {
      const div = document.createElement('div');
      document.body.appendChild(div);

      // カメラ選択用のセレクトボックスを追加
      const select_div = document.createElement('div');
      select_div.style.marginBottom = '10px';
      const select_label = document.createElement('label');
      select_label.textContent = 'カメラを選択: ';
      const cam_select = document.createElement('select');
      select_div.appendChild(select_label);
      select_div.appendChild(cam_select);
      div.appendChild(select_div);

      // video element
      const video = document.createElement('video');
      video.style.display = 'None';
      div.appendChild(video);

      let stream = null;

      // カメラリストを取得してセレクトボックスに初期化する関数
      async function initCameraSelect() {
        // 初回のみ権限許可を得るためにダミーのストリームを取得
        const initStream = await navigator.mediaDevices.getUserMedia({video: true});
        initStream.getTracks().forEach(track => track.stop());

        const devices = await navigator.mediaDevices.enumerateDevices();
        const videoDevices = devices.filter(device => device.kind === 'videoinput');

        cam_select.innerHTML = '';
        videoDevices.forEach((device, index) => {
          const option = document.createElement('option');
          option.value = device.deviceId;
          option.text = device.label || `Camera ${index + 1}`;
          cam_select.appendChild(option);
        });
      }

      // 指定したデバイスIDでカメラを開始する関数
      async function startCamera(deviceId) {
        if (stream) {
          stream.getVideoTracks().forEach(track => track.stop());
        }
        const constraints = {
          video: deviceId ? { deviceId: { exact: deviceId } } : true
        };
        stream = await navigator.mediaDevices.getUserMedia(constraints);
        video.srcObject = stream;
        await video.play();
      }

      // 初期設定
      await initCameraSelect();
      if (cam_select.options.length > 0) {
        await startCamera(cam_select.value);
      } else {
        await startCamera(null);
      }

      // カメラが切り替えられた時のイベント
      cam_select.onchange = async () => {
        await startCamera(cam_select.value);
      };

      // canvas for display. frame rate is depending on display size and jpeg quality.
      display_size = 500
      const src_canvas = document.createElement('canvas');
      src_canvas.width  = display_size;
      src_canvas.height = display_size * (video.videoHeight || display_size) / (video.videoWidth || display_size);
      const src_canvasCtx = src_canvas.getContext('2d');
      src_canvasCtx.translate(src_canvas.width, 0);
      src_canvasCtx.scale(-1, 1);
      div.appendChild(src_canvas);

      const dst_canvas = document.createElement('canvas');
      dst_canvas.width  = src_canvas.width;
      dst_canvas.height = src_canvas.height;
      const dst_canvasCtx = dst_canvas.getContext('2d');
      div.appendChild(dst_canvas);

      // exit button
      const btn_div = document.createElement('div');
      document.body.appendChild(btn_div);
      const exit_btn = document.createElement('button');
      exit_btn.textContent = 'Exit';
      var exit_flg = true
      exit_btn.onclick = function() {exit_flg = false};
      btn_div.appendChild(exit_btn);

      // Resize the output to fit the video element.
      google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);

      var send_num = 0
      // loop
      _canvasUpdate();
      async function _canvasUpdate() {
            // カメラ切り替え時にサイズが変わる可能性があるため動的にキャンバスサイズを調整
            if (video.videoWidth && video.videoHeight) {
              const new_height = display_size * video.videoHeight / video.videoWidth;
              if (src_canvas.height !== new_height) {
                src_canvas.height = new_height;
                dst_canvas.height = new_height;
                src_canvasCtx.translate(src_canvas.width, 0);
                src_canvasCtx.scale(-1, 1);
              }
              src_canvasCtx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight, 0, 0, src_canvas.width, src_canvas.height);
            }

            if (send_num<1 && video.videoWidth){
                send_num += 1
                const img = src_canvas.toDataURL('image/jpeg', quality);
                const result = google.colab.kernel.invokeFunction('notebook.run', [img], {});
                result.then(function(value) {
                    parse = JSON.parse(JSON.stringify(value))["data"]
                    parse = JSON.parse(JSON.stringify(parse))["application/json"]
                    parse = JSON.parse(JSON.stringify(parse))["img_str"]
                    var image = new Image()
                    image.src = parse;
                    image.onload = function(){dst_canvasCtx.drawImage(image, 0, 0)}
                    send_num -= 1
                })
            }
            if (exit_flg){
                requestAnimationFrame(_canvasUpdate);
            }else{
                if (stream) stream.getVideoTracks()[0].stop();
            }
      };
    }
    ''')
  display(js)
  data = eval_js('useCam({})'.format(quality))

Run the program

use_cam()

・ Please grant permission when a request for access is made.
・ Click "Exit" at the bottom left to stop.

3. Let's try using YOLO

Install

!pip install ultralytics

Import

import cv2
from ultralytics import YOLO

# Load the YOLO model
#model = YOLO("best.pt")
yolo_model = YOLO("yolo11n.pt")

Fully automated object detection

def run(img_str):
    #decode to image
    decimg = base64.b64decode(img_str.split(',')[1], validate=True)
    decimg = Image.open(BytesIO(decimg))
    decimg = np.array(decimg, dtype=np.uint8);
    decimg = cv2.cvtColor(decimg, cv2.COLOR_BGR2RGB)
    gray = cv2.cvtColor(decimg, cv2.COLOR_BGR2GRAY)

    #############your process###############
    # YOLO inference
    results = yolo_model.predict(decimg, imgsz=640, verbose=False)
    # YOLO returns results as a list, so extract them
    result = results[0]
    # Drow bounding boxes (using YOLO built-in function)
    annotated_img = result.plot()   # numpy array (RGB)
    # Resize if necessary
    # annotated_img = cv2.resize(annotated_img, (w//2, h//2))
    out_img = annotated_img
    #############your process###############
    #encode to string
    _, encimg = cv2.imencode(".jpg", out_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
    #img_str = encimg.tostring()
    img_str = encimg.tobytes()
    img_str = "data:image/jpeg;base64," + base64.b64encode(img_str).decode('utf-8')
    return IPython.display.JSON({'img_str': img_str})
output.register_callback('notebook.run', run)

Run the program

use_cam()

4. Set to manual result display

Object detection

def run(img_str):
    #---------------------------------------------
    decimg = base64.b64decode(img_str.split(',')[1], validate=True)
    decimg = Image.open(BytesIO(decimg))
    frame = np.array(decimg, dtype=np.uint8)
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    #---------------------------------------------
    results = yolo_model(frame, conf=0.5, iou=0.5, verbose=False)
    items = results[0]
    annotated_frame = frame.copy()
    #---------------------------------------------
    # Draw bounding boxes manually
    #---------------------------------------------
    for item in items:
        # class ID
        cls = int(item.boxes.cls)
        label = item.names[cls]  # Class name

        # Score
        score = float(item.boxes.conf.cpu().numpy()[0])

        # bbox coordinates (x1, y1, x2, y2)
        x1, y1, x2, y2 = item.boxes.xyxy.cpu().numpy()[0]

        # Tracking ID (None if not available)
        id_value = item.boxes.id
        track_id = '' if id_value is None else int(id_value.cpu().numpy()[0])

        # Bounding boxes
        cv2.rectangle(
            annotated_frame,
            (int(x1), int(y1)),
            (int(x2), int(y2)),
            (0, 255, 0),
            2
        )

        # Text label
        if track_id == '':
            text = f"{label}: {score:.2f}"
        else:
            text = f"{track_id} {label}: {score:.2f}"

        cv2.putText(
            annotated_frame,
            text,
            (int(x1), int(y1) - 10),
            cv2.FONT_HERSHEY_SIMPLEX,
            0.75,
            (0, 255, 0),
            2
        )

    #---------------------------------------------
    _, encimg = cv2.imencode(".jpg", annotated_frame, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
    img_bytes = encimg.tobytes()
    img_str = "data:image/jpeg;base64," + base64.b64encode(img_bytes).decode('utf-8')

    return IPython.display.JSON({'img_str': img_str})
    
output.register_callback('notebook.run', run)

Run the program

use_cam()

5. Add tracking functionality

Train the model

def run(img_str):
    # Base64 → Image
    decimg = base64.b64decode(img_str.split(',')[1], validate=True)
    decimg = Image.open(BytesIO(decimg))
    frame = np.array(decimg, dtype=np.uint8)
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    # Built-in YOLO11n tracking
    results = yolo_model.track(frame, conf=0.5, iou=0.5, verbose=False,
                               show_labels=False, show_boxes=False, show_conf=False)
    items = results[0]
    annotated = frame.copy()

    for item in items:
        cls = int(item.boxes.cls)
        label = item.names[cls]
        score = float(item.boxes.conf.cpu().numpy()[0])
        x1, y1, x2, y2 = item.boxes.xyxy.cpu().numpy()[0]

        track_id = item.boxes.id
        track_id_int = '' if track_id is None else int(track_id.cpu())

        # Draw
        cv2.rectangle(annotated, (int(x1), int(y1)), (int(x2), int(y2)), (0,255,0), 2)
        cv2.putText(annotated, f"ID:{track_id_int} {label}: {score:.2f}",
                    (int(x1), int(y1)-10), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0,255,0), 2)

    # Convert back to Base64 and return
    _, encimg = cv2.imencode(".jpg", annotated, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
    img_bytes = encimg.tobytes()
    img_str = "data:image/jpeg;base64," + base64.b64encode(img_bytes).decode('utf-8')

    return IPython.display.JSON({'img_str': img_str})

output.register_callback('notebook.run', run)

Run the program

use_cam()

6. Pose estimation using YOLO

6.1 Fully automated version

Fully automated version

import base64
import IPython
from io import BytesIO
from PIL import Image
import numpy as np
import cv2
from ultralytics import YOLO

model = YOLO("yolo11n-pose.pt")

def run_pose(img_str):
    """
    Perform YOLO11n-pose inference on camera input in Colab
    Draw pose keypoints and skeleton, then return the result
    """

    # -----------------------------
    # ① Base64 → Image
    # -----------------------------
    decimg = base64.b64decode(img_str.split(',')[1], validate=True)
    decimg = Image.open(BytesIO(decimg))
    frame = np.array(decimg, dtype=np.uint8)
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    # -----------------------------
    # ② YOLO11n-pose inference
    # -----------------------------
    results = model(frame, conf=0.5, iou=0.3, verbose=False)
    annotated_frame = results[0].plot()  # Image with keypoints and skeleton drawn

    # -----------------------------
    # ③ Convert back to Base64 and return
    # -----------------------------
    _, encimg = cv2.imencode(".jpg", annotated_frame, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
    img_bytes = encimg.tobytes()
    img_str = "data:image/jpeg;base64," + base64.b64encode(img_bytes).decode('utf-8')

    return IPython.display.JSON({'img_str': img_str})

output.register_callback('notebook.run', run_pose)

Run the program

use_cam()

6.2 Manual result display version

Manual result display version

import base64
import IPython
from io import BytesIO
from PIL import Image
import numpy as np
import cv2
from ultralytics import YOLO

model = YOLO("yolo11n-pose.pt")

# Keypoint names
KEYPOINTS_NAMES = [
    "nose", "eye(L)", "eye(R)", "ear(L)", "ear(R)",
    "shoulder(L)", "shoulder(R)", "elbow(L)", "elbow(R)",
    "wrist(L)", "wrist(R)", "hip(L)", "hip(R)",
    "knee(L)", "knee(R)", "ankle(L)", "ankle(R)"
]

def run_pose_manual(img_str):
    """
    Perform YOLO11n-pose inference manually in Colab
    Draw keypoints, skeleton, part names, and bounding boxes
    """

    # -----------------------------
    # ① Base64 → Image
    # -----------------------------
    decimg = base64.b64decode(img_str.split(',')[1], validate=True)
    decimg = Image.open(BytesIO(decimg))
    frame = np.array(decimg, dtype=np.uint8)
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    annotated_frame = frame.copy()

    # -----------------------------
    # ② YOLO11n-pose inference
    # -----------------------------
    results = model(frame, conf=0.5, iou=0.3, verbose=False)
    items = results[0]

    for item in items:
        # Bounding box
        x1, y1, x2, y2 = item.boxes.xyxy.cpu().numpy()[0]
        score = float(item.boxes.conf.cpu().numpy()[0])
        cls = int(item.boxes.cls)
        label = item.names[cls]

        # Tracking ID
        id_value = item.boxes.id
        track_ids = '' if id_value is None else int(item.boxes.id.cpu())

        # Draw bbox
        cv2.rectangle(annotated_frame, (int(x1), int(y1)), (int(x2), int(y2)), (0,255,0), 2)
        text = f"ID:{track_ids} {label}: {score:.2f}" if track_ids else f"{label}: {score:.2f}"
        cv2.putText(annotated_frame, text, (int(x1), int(y1)-10),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0,255,0), 2)

        #  Draw keypoints
        keypoints = results[0].keypoints
        xys = keypoints.xy[0].tolist()       # Coordinates
        confs = keypoints.conf[0].tolist()   # Confidence

        for idx, (xy, kscore) in enumerate(zip(xys, confs)):
            if kscore < 0.5:
                continue
            x, y = int(xy[0]), int(xy[1])

            # Purple rectangle
            cv2.rectangle(annotated_frame, (x, y), (x+3, y+3), (255,0,255), cv2.FILLED)
            # Part name
            cv2.putText(annotated_frame, KEYPOINTS_NAMES[idx], (x+5, y),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255,0,255), 1, cv2.LINE_AA)

    # -----------------------------
    # ③ Convert back to Base64 and return
    # -----------------------------
    _, encimg = cv2.imencode(".jpg", annotated_frame, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
    img_bytes = encimg.tobytes()
    img_str_out = "data:image/jpeg;base64," + base64.b64encode(img_bytes).decode('utf-8')

    return IPython.display.JSON({'img_str': img_str_out})

output.register_callback('notebook.run', run_pose_manual)

Run the program

use_cam()

Exercise

In the pose estimation program from section "6.2", modify the code so that when a person with a specific pose is detected, a corresponding message is displayed. (e.g., display "Hand raised" when someone is raising their hand)