はじめに

DeepLabのセマンティックセグメンテーションをやっていきます

システム環境

Windows10（RTX2080 Max-Q、i7-8750H、RAM16GB）
Anaconda 2020.02
Python 3.6
CUDA Toolkit v10.0

導入

modelsをクローンします。
DeepLabはresearch/deeplabにあります。deeplab_demo.ipynbを実行してみましょう。

まずは、deeplab環境を作成します。

$ conda create -n deeplab python=3.6
$ conda activate deeplab
$ pip install jupyter
$ pip install matplotlib
$ pip install pillow
$ pip install tensorflow-gpu==1.15

CUDA Toolkit v10.0をインストールしてください。cudart64_100.dllが必要です。システム環境変数にパスを通すか、プロジェクトと同じディレクトリにコピペする必要があります。見つからない場合はCPUで実行されると思います。

cudnn64_7.dllも必要なので、cuDNN DownloadからDownload cuDNN v7.6.5 (November 5th, 2019), for CUDA 10.0のcuDNN Library for Windows 10をダウンロードしてください。以下のファイルをC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0にコピペします。

jupyter notebookを開きます。

$ cd models-master/research/deeplab
$ jupyter notebook

出力されたURLにアクセスして、deeplab_demo.ipynbを開きます。上から順番に実行していきましょう。
最初のセルの%tensorflow_version 1.x はエラーが出たので、コメントアウトしました。

import os
from io import BytesIO
import tarfile
import tempfile
from six.moves import urllib

from matplotlib import gridspec
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image

# %tensorflow_version 1.x
import tensorflow as tf

実行結果はこちらです。

プログラムの部分をコピペして、deeplab_demo.pyを作成し、実行してみましょう。同じことができると思います。

ウェブカメラの映像を入力

ウェブカメラから読み込んで処理してみましょう。
描画にはOpenCVを用いるので、ライブラリをインストールします。

$ pip install opencv-python

次のプログラムを作成し、実行します。

import os
from io import BytesIO
import tarfile
import tempfile
from six.moves import urllib

from matplotlib import gridspec
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image

# %tensorflow_version 1.x
import tensorflow as tf


class DeepLabModel(object):
  """Class to load deeplab model and run inference."""

  INPUT_TENSOR_NAME = 'ImageTensor:0'
  OUTPUT_TENSOR_NAME = 'SemanticPredictions:0'
  INPUT_SIZE = 513
  FROZEN_GRAPH_NAME = 'frozen_inference_graph'

  def __init__(self, tarball_path):
    """Creates and loads pretrained deeplab model."""
    self.graph = tf.Graph()

    graph_def = None
    # Extract frozen graph from tar archive.
    tar_file = tarfile.open(tarball_path)
    for tar_info in tar_file.getmembers():
      if self.FROZEN_GRAPH_NAME in os.path.basename(tar_info.name):
        file_handle = tar_file.extractfile(tar_info)
        graph_def = tf.GraphDef.FromString(file_handle.read())
        break

    tar_file.close()

    if graph_def is None:
      raise RuntimeError('Cannot find inference graph in tar archive.')

    with self.graph.as_default():
      tf.import_graph_def(graph_def, name='')

    self.sess = tf.Session(graph=self.graph)

  def run(self, image):
    """Runs inference on a single image.

    Args:
      image: A PIL.Image object, raw input image.

    Returns:
      resized_image: RGB image resized from original input image.
      seg_map: Segmentation map of `resized_image`.
    """
    width, height = image.size
    resize_ratio = 1.0 * self.INPUT_SIZE / max(width, height)
    target_size = (int(resize_ratio * width), int(resize_ratio * height))
    resized_image = image.convert('RGB').resize(target_size, Image.ANTIALIAS)
    batch_seg_map = self.sess.run(
        self.OUTPUT_TENSOR_NAME,
        feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]})
    seg_map = batch_seg_map[0]
    return resized_image, seg_map


def create_pascal_label_colormap():
  """Creates a label colormap used in PASCAL VOC segmentation benchmark.

  Returns:
    A Colormap for visualizing segmentation results.
  """
  colormap = np.zeros((256, 3), dtype=int)
  ind = np.arange(256, dtype=int)

  for shift in reversed(range(8)):
    for channel in range(3):
      colormap[:, channel] |= ((ind >> channel) & 1) << shift
    ind >>= 3

  return colormap


def label_to_color_image(label):
  """Adds color defined by the dataset colormap to the label.

  Args:
    label: A 2D array with integer type, storing the segmentation label.

  Returns:
    result: A 2D array with floating type. The element of the array
      is the color indexed by the corresponding element in the input label
      to the PASCAL color map.

  Raises:
    ValueError: If label is not of rank 2 or its value is larger than color
      map maximum entry.
  """
  if label.ndim != 2:
    raise ValueError('Expect 2-D input label')

  colormap = create_pascal_label_colormap()

  if np.max(label) >= len(colormap):
    raise ValueError('label value too large.')

  return colormap[label]


def vis_segmentation(image, seg_map):
  """Visualizes input image, segmentation map and overlay view."""
  plt.figure(figsize=(15, 5))
  grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1])

  plt.subplot(grid_spec[0])
  plt.imshow(image)
  plt.axis('off')
  plt.title('input image')

  plt.subplot(grid_spec[1])
  seg_image = label_to_color_image(seg_map).astype(np.uint8)
  plt.imshow(seg_image)
  plt.axis('off')
  plt.title('segmentation map')

  plt.subplot(grid_spec[2])
  plt.imshow(image)
  plt.imshow(seg_image, alpha=0.7)
  plt.axis('off')
  plt.title('segmentation overlay')

  unique_labels = np.unique(seg_map)
  ax = plt.subplot(grid_spec[3])
  plt.imshow(
      FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest')
  ax.yaxis.tick_right()
  plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])
  plt.xticks([], [])
  ax.tick_params(width=0.0)
  plt.grid('off')
  plt.show()

LABEL_NAMES = np.asarray([
    'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus',
    'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike',
    'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tv'
])

FULL_LABEL_MAP = np.arange(len(LABEL_NAMES)).reshape(len(LABEL_NAMES), 1)
FULL_COLOR_MAP = label_to_color_image(FULL_LABEL_MAP)

# @param ['mobilenetv2_coco_voctrainaug', 'mobilenetv2_coco_voctrainval', 'xception_coco_voctrainaug', 'xception_coco_voctrainval']
MODEL_NAME = 'mobilenetv2_coco_voctrainaug'

_DOWNLOAD_URL_PREFIX = 'http://download.tensorflow.org/models/'
_MODEL_URLS = {
    'mobilenetv2_coco_voctrainaug':
        'deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz',
    'mobilenetv2_coco_voctrainval':
        'deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz',
    'xception_coco_voctrainaug':
        'deeplabv3_pascal_train_aug_2018_01_04.tar.gz',
    'xception_coco_voctrainval':
        'deeplabv3_pascal_trainval_2018_01_04.tar.gz',
}
_TARBALL_NAME = 'deeplab_model.tar.gz'

model_dir = tempfile.mkdtemp()
tf.gfile.MakeDirs(model_dir)

download_path = os.path.join(model_dir, _TARBALL_NAME)
print('downloading model, this might take a while...')
urllib.request.urlretrieve(_DOWNLOAD_URL_PREFIX + _MODEL_URLS[MODEL_NAME],
                           download_path)
print('download completed! loading DeepLab model...')

MODEL = DeepLabModel(download_path)
print('model loaded successfully!')


SAMPLE_IMAGE = 'image1'  # @param ['image1', 'image2', 'image3']
IMAGE_URL = ''  # @param {type:"string"}

_SAMPLE_URL = ('https://github.com/tensorflow/models/blob/master/research/'
               'deeplab/g3doc/img/%s.jpg?raw=true')


def run_visualization(url):
  """Inferences DeepLab model and visualizes result."""
  try:
    f = urllib.request.urlopen(url)
    jpeg_str = f.read()
    original_im = Image.open(BytesIO(jpeg_str))
  except IOError:
    print('Cannot retrieve image. Please check url: ' + url)
    return

  print('running deeplab on image %s...' % url)
  resized_im, seg_map = MODEL.run(original_im)

  vis_segmentation(resized_im, seg_map)

# image_url = IMAGE_URL or _SAMPLE_URL % SAMPLE_IMAGE
# run_visualization(image_url)

import cv2 

def vis_segmentation_opencv(image, seg_map):
    seg_image = label_to_color_image(seg_map).astype(np.uint8)
    unique_labels = np.unique(seg_map)
    return seg_image, unique_labels

cap = cv2.VideoCapture(0)
while True:
    ret, frame = cap.read()
    if ret:
        resized_im, seg_map = MODEL.run(
            Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)))
        seg_image, unique_labels = vis_segmentation_opencv(resized_im, seg_map)
        seg_image_ = np.array(seg_image, dtype=np.uint8)
        h, w, ch = frame.shape
        resized_seg_im = cv2.resize(seg_image_, (w, h), interpolation=cv2.INTER_NEAREST)
        output = (0.4*resized_seg_im + 0.6*frame).astype("uint8")
        cv2.imshow('output', output)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    else:
        cap.set(cv2.CAP_PROP_POS_FRAMES, 0)

結果はこちらになります。

DeepLabによるSemantic Segmentation #DeepLab #CV #Python #OpenCV pic.twitter.com/XBF0pdzB9E
— 藤本賢志(ガチ本)@pixivFANBOXはじめました (@sotongshi) June 19, 2020

qキーでウィンドウを閉じることができます。

お疲れ様でした。