Edited at

Keras学習済みモデルをFine-tuningさせて精度比較


概要

各種Keras学習済みモデルをFruits 360データセットでFine-tuningさせ、クラス分類モデルを構築

モデル精度を比較する

対象モデル

- VGG16

- InceptionV3

- Xception

- MobileNet

Fruits 360

https://www.kaggle.com/moltean/fruits

- KaggleのFruits 360 dataset

- 81クラス、55244枚の画像データセット

実行環境

Google Colaboratory(GPU)

import numpy as np

import pandas as pd
import matplotlib.pyplot as plt
%config InlineBackend.figure_formats = {'png', 'retina'}

import os, cv2, zipfile, io, re, glob

from PIL import Image
from sklearn.model_selection import train_test_split
from keras.applications.vgg16 import VGG16
from keras.applications.inception_v3 import InceptionV3
from keras.applications.inception_resnet_v2 import InceptionResNetV2
from keras.applications.xception import Xception
from keras.models import Model, load_model
from keras.layers.core import Dense
from keras.layers.pooling import GlobalAveragePooling2D
from keras.optimizers import Adam, RMSprop, SGD
from keras.utils.np_utils import to_categorical
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator


データ取得

81クラスの内、30クラスの画像を取得。

# ZIP読み込み

z = zipfile.ZipFile('../dataset/fruits-360.zip')

# ラベリングされたディレクトリのみ取得

img_dirs = [ x for x in z.namelist() if re.search("^fruits-360/Training/.*/$", x)]
# 不要な文字列削除
img_dirs = [ x.replace('fruits-360/Training/', '') for x in img_dirs]
img_dirs = [ x.replace('/', '') for x in img_dirs]
img_dirs.sort()

# クラス取得
classes = img_dirs

# クラス数
#num_classes = len(classes)
num_classes = 30

del img_dirs

画像は150にリサイズ後、配列に変換

# 画像サイズ

image_size = 150

# 画像を取得し、配列に変換
def im2array(path):
X = []
y = []
class_num = 0

for class_name in classes:
if class_num == num_classes : break
imgfiles = [ x for x in z.namelist() if re.search("^" + path + class_name + "/.*jpg$", x)]
for imgfile in imgfiles:
# ZIPから画像読み込み
image = Image.open(io.BytesIO(z.read(imgfile)))
# RGB変換
image = image.convert('RGB')
# リサイズ
image = image.resize((image_size, image_size))
# 画像から配列に変換
data = np.asarray(image)
X.append(data)
y.append(classes.index(class_name))
class_num += 1

X = np.array(X)
y = np.array(y)

return X, y

trainデータ取得

%%time

X_train, y_train = im2array("fruits-360/Training/")
print(X_train.shape, y_train.shape)

(15094, 150, 150, 3) (15094,)

CPU times: user 14.1 s, sys: 1.56 s, total: 15.7 s

Wall time: 15.9 s

testデータ取得

%%time

X_test, y_test = im2array("fruits-360/Test/")
print(X_test.shape, y_test.shape)

(5060, 150, 150, 3) (5060,)

CPU times: user 8.02 s, sys: 307 ms, total: 8.32 s

Wall time: 8.39 s

del z

# データ型の変換

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

# 正規化
X_train /= 255
X_test /= 255

# one-hot 変換
y_train = to_categorical(y_train, num_classes = num_classes)
y_test = to_categorical(y_test, num_classes = num_classes)
print(y_train.shape, y_test.shape)

(15094, 30) (5060, 30)

trainデータからvalidデータを分割

X_train, X_valid, y_train, y_valid = train_test_split(

X_train,
y_train,
random_state = 0,
stratify = y_train,
test_size = 0.2
)
print(X_train.shape, y_train.shape, X_valid.shape, y_valid.shape)

(12075, 150, 150, 3) (12075, 30) (3019, 150, 150, 3) (3019, 30)


学習済みモデルの精度比較

Data Augmentation

datagen = ImageDataGenerator(

featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
rotation_range = 0,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = True,
vertical_flip = False
)

Callback

# EarlyStopping

early_stopping = EarlyStopping(
monitor = 'val_loss',
patience = 10,
verbose = 1
)

# ModelCheckpoint
weights_dir = './weights/'
if os.path.exists(weights_dir) == False:os.mkdir(weights_dir)
model_checkpoint = ModelCheckpoint(
weights_dir + "val_loss{val_loss:.3f}.hdf5",
monitor = 'val_loss',
verbose = 1,
save_best_only = True,
save_weights_only = True,
period = 3
)

# reduce learning rate
reduce_lr = ReduceLROnPlateau(
monitor = 'val_loss',
factor = 0.1,
patience = 3,
verbose = 1
)

# log for TensorBoard
logging = TensorBoard(log_dir = "log/")

各種関数定義

# モデル学習

def model_fit():
hist = model.fit_generator(
datagen.flow(X_train, y_train, batch_size = 32),
steps_per_epoch = X_train.shape[0] // 32,
epochs = 50,
validation_data = (X_valid, y_valid),
callbacks = [early_stopping, reduce_lr],
shuffle = True,
verbose = 1
)
return hist

# モデル保存
model_dir = './model/'
if os.path.exists(model_dir) == False : os.mkdir(model_dir)

def model_save(model_name):
model.save(model_dir + 'model_' + model_name + '.hdf5')

# optimizerのない軽量モデルを保存(学習や評価不可だが、予測は可能)
model.save(model_dir + 'model_' + model_name + '-opt.hdf5', include_optimizer = False)

# 学習曲線をプロット
def learning_plot(title):
plt.figure(figsize = (18,6))

# accuracy
plt.subplot(1, 2, 1)
plt.plot(hist.history["acc"], label = "acc", marker = "o")
plt.plot(hist.history["val_acc"], label = "val_acc", marker = "o")
#plt.yticks(np.arange())
#plt.xticks(np.arange())
plt.ylabel("accuracy")
plt.xlabel("epoch")
plt.title(title)
plt.legend(loc = "best")
plt.grid(color = 'gray', alpha = 0.2)

# loss
plt.subplot(1, 2, 2)
plt.plot(hist.history["loss"], label = "loss", marker = "o")
plt.plot(hist.history["val_loss"], label = "val_loss", marker = "o")
#plt.yticks(np.arange())
#plt.xticks(np.arange())
plt.ylabel("loss")
plt.xlabel("epoch")
plt.title(title)
plt.legend(loc = "best")
plt.grid(color = 'gray', alpha = 0.2)

plt.show()

# モデル評価
def model_evaluate():
score = model.evaluate(X_test, y_test, verbose = 1)
print("evaluate loss: {[0]:.4f}".format(score))
print("evaluate acc: {[1]:.1%}".format(score))


VGG16

base_model = VGG16(

include_top = False,
weights = "imagenet",
input_shape = None
)

# 全結合層の新規構築
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation = 'relu')(x)
predictions = Dense(num_classes, activation = 'softmax')(x)

# ネットワーク定義
model = Model(inputs = base_model.input, outputs = predictions)
print("{}層".format(len(model.layers)))

22層

全22層の内、17層までfreezeさせ、18層以降を学習させる。

# 17層までfreeze

for layer in model.layers[:17]:
layer.trainable = False

# 18層以降、学習させる
for layer in model.layers[17:]:
layer.trainable = True

# layer.trainableの設定後にcompile
model.compile(
optimizer = Adam(),
loss = 'categorical_crossentropy',
metrics = ["accuracy"]
)

%%time

hist = model_fit()

Epoch 1/50

377/377 [==============================] - 98s 260ms/step - loss: 0.4985 - acc: 0.8490 - val_loss: 0.1256 - val_acc: 0.9530
Epoch 2/50
377/377 [==============================] - 96s 255ms/step - loss: 0.0727 - acc: 0.9732 - val_loss: 0.0732 - val_acc: 0.9665
Epoch 3/50
377/377 [==============================] - 96s 255ms/step - loss: 0.0581 - acc: 0.9770 - val_loss: 0.0297 - val_acc: 0.9851

〜省略〜

Epoch 24/50
377/377 [==============================] - 94s 248ms/step - loss: 0.0203 - acc: 0.9871 - val_loss: 0.0218 - val_acc: 0.9834
Epoch 25/50
377/377 [==============================] - 93s 247ms/step - loss: 0.0206 - acc: 0.9854 - val_loss: 0.0218 - val_acc: 0.9834
Epoch 26/50
377/377 [==============================] - 93s 246ms/step - loss: 0.0204 - acc: 0.9864 - val_loss: 0.0218 - val_acc: 0.9834

Epoch 00026: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.
Epoch 27/50
377/377 [==============================] - 92s 245ms/step - loss: 0.0203 - acc: 0.9872 - val_loss: 0.0218 - val_acc: 0.9834
Epoch 00027: early stopping
CPU times: user 50min 41s, sys: 4min 41s, total: 55min 22s
Wall time: 42min 35s

learning_plot("VGG16")

model_evaluate()

5060/5060 [==============================] - 24s 5ms/step

evaluate loss: 0.1116

evaluate acc: 96.1%

model_save("VGG16")


InceptionV3

base_model = InceptionV3(

include_top = False,
weights = "imagenet",
input_shape = None
)

# 全結合層の新規構築
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation = 'relu')(x)
predictions = Dense(num_classes, activation = 'softmax')(x)

# ネットワーク定義
model = Model(inputs = base_model.input, outputs = predictions)
print("{}層".format(len(model.layers)))

314層

全314層の内、249層までfreezeさせ、250層以降を学習させる。

# 249層までfreeze

for layer in model.layers[:249]:
layer.trainable = False

# Batch Normalization の freeze解除
if layer.name.startswith('batch_normalization'):
layer.trainable = True

#250層以降、学習させる
for layer in model.layers[249:]:
layer.trainable = True

# layer.trainableの設定後にcompile
model.compile(
optimizer = Adam(),
loss = 'categorical_crossentropy',
metrics = ["accuracy"]
)

%%time

hist=model_fit()

Epoch 1/50

377/377 [==============================] - 148s 394ms/step - loss: 0.3223 - acc: 0.9127 - val_loss: 0.0609 - val_acc: 0.9748
Epoch 2/50
377/377 [==============================] - 129s 343ms/step - loss: 0.0962 - acc: 0.9684 - val_loss: 0.0469 - val_acc: 0.9781
Epoch 3/50
377/377 [==============================] - 129s 343ms/step - loss: 0.0676 - acc: 0.9756 - val_loss: 0.0365 - val_acc: 0.9821

〜省略〜

Epoch 19/50
377/377 [==============================] - 129s 341ms/step - loss: 0.0225 - acc: 0.9859 - val_loss: 0.0210 - val_acc: 0.9844
Epoch 20/50
377/377 [==============================] - 128s 340ms/step - loss: 0.0235 - acc: 0.9857 - val_loss: 0.0210 - val_acc: 0.9844
Epoch 21/50
377/377 [==============================] - 129s 342ms/step - loss: 0.0210 - acc: 0.9865 - val_loss: 0.0210 - val_acc: 0.9848

Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.
Epoch 00021: early stopping
CPU times: user 1h 8min 40s, sys: 9min 16s, total: 1h 17min 56s
Wall time: 45min 28s

learning_plot("InceptionV3")

model_evaluate()

5060/5060 [==============================] - 15s 3ms/step

evaluate loss: 0.2082

evaluate acc: 96.5%

model_save("InceptionV3")

VGG16に比べ、accuracyは良くなったが、代わりにlossが増え、必ずしも精度が高いとは言い切れない。


Xception

base_model = Xception(

include_top = False,
weights = "imagenet",
input_shape = None
)

# 全結合層の新規構築
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation = 'relu')(x)
predictions = Dense(num_classes, activation = 'softmax')(x)

# ネットワーク定義
model = Model(inputs = base_model.input, outputs = predictions)
print("{}層".format(len(model.layers)))

135層

全135層の内、108層までfreezeさせ、109層以降を学習させる。

#108層までfreeze

for layer in model.layers[:108]:
layer.trainable = False

# Batch Normalization の freeze解除
if layer.name.startswith('batch_normalization'):
layer.trainable = True
if layer.name.endswith('bn'):
layer.trainable = True

#109層以降、学習させる
for layer in model.layers[108:]:
layer.trainable = True

# layer.trainableの設定後にcompile
model.compile(
optimizer = Adam(),
loss = 'categorical_crossentropy',
metrics = ["accuracy"]
)

%%time

hist = model_fit()

Epoch 1/50

377/377 [==============================] - 172s 455ms/step - loss: 0.2736 - acc: 0.9318 - val_loss: 0.0859 - val_acc: 0.9669
Epoch 2/50
377/377 [==============================] - 162s 429ms/step - loss: 0.1284 - acc: 0.9626 - val_loss: 0.0389 - val_acc: 0.9824
Epoch 3/50
377/377 [==============================] - 161s 426ms/step - loss: 0.0882 - acc: 0.9724 - val_loss: 0.0275 - val_acc: 0.9824

〜省略〜

Epoch 16/50
377/377 [==============================] - 161s 427ms/step - loss: 0.0220 - acc: 0.9862 - val_loss: 0.0232 - val_acc: 0.9838
Epoch 17/50
377/377 [==============================] - 161s 427ms/step - loss: 0.0219 - acc: 0.9864 - val_loss: 0.0229 - val_acc: 0.9841
Epoch 18/50
377/377 [==============================] - 161s 427ms/step - loss: 0.0228 - acc: 0.9859 - val_loss: 0.0226 - val_acc: 0.9844

Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.
Epoch 19/50
377/377 [==============================] - 161s 427ms/step - loss: 0.0229 - acc: 0.9865 - val_loss: 0.0226 - val_acc: 0.9844
Epoch 00019: early stopping
CPU times: user 1h 41s, sys: 12min 51s, total: 1h 13min 33s
Wall time: 51min 20s

learning_plot("Xception")

model_evaluate()

5060/5060 [==============================] - 25s 5ms/step

evaluate loss: 0.0977

evaluate acc: 97.7%

model_save("Xception")

VGG16やInceptionV3に比べ、accuracy、lossともに精度が高い。


MobileNet

base_model = MobileNet(

include_top = False,
weights = "imagenet",
input_shape = None
)

# 全結合層の新規構築
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation = 'relu')(x)
predictions = Dense(num_classes, activation = 'softmax')(x)

# ネットワーク定義
model = Model(inputs = base_model.input, outputs = predictions)
print("{}層".format(len(model.layers)))

90層

全90層の内、72層までfreezeさせ、73層以降を学習させる。

# 72層までfreeze

for layer in model.layers[:72]:
layer.trainable = False

# Batch Normalization の freeze解除
if "bn" in layer.name:
layer.trainable = True

#73層以降、学習させる
for layer in model.layers[72:]:
layer.trainable = True

# layer.trainableの設定後にcompile
model.compile(
optimizer = Adam(),
loss = 'categorical_crossentropy',
metrics = ["accuracy"]
)

%%time

hist = model_fit()

Epoch 1/50

377/377 [==============================] - 92s 244ms/step - loss: 0.0857 - acc: 0.9759 - val_loss: 0.2791 - val_acc: 0.9543
Epoch 2/50
377/377 [==============================] - 79s 210ms/step - loss: 0.0938 - acc: 0.9746 - val_loss: 0.5770 - val_acc: 0.9232
Epoch 3/50
377/377 [==============================] - 80s 212ms/step - loss: 0.0881 - acc: 0.9758 - val_loss: 0.1943 - val_acc: 0.9629

〜省略〜

Epoch 19/50
377/377 [==============================] - 82s 217ms/step - loss: 0.0209 - acc: 0.9859 - val_loss: 0.0212 - val_acc: 0.9854
Epoch 20/50
377/377 [==============================] - 81s 216ms/step - loss: 0.0204 - acc: 0.9867 - val_loss: 0.0212 - val_acc: 0.9858
Epoch 21/50
377/377 [==============================] - 81s 216ms/step - loss: 0.0197 - acc: 0.9867 - val_loss: 0.0212 - val_acc: 0.9854

Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.
Epoch 22/50
377/377 [==============================] - 82s 216ms/step - loss: 0.0216 - acc: 0.9869 - val_loss: 0.0212 - val_acc: 0.9858
Epoch 00022: early stopping
CPU times: user 42min 23s, sys: 3min 30s, total: 45min 53s
Wall time: 29min 59s

learning_plot("MobileNet")

model_evaluate()

5060/5060 [==============================] - 7s 1ms/step

evaluate loss: 0.1705

evaluate acc: 97.6%

model_save("MobileNet")

エポックごとの学習速度が、最も早い。

データセットに依るのだろうが、accuracyがVGG16やInceptionV3よりも高く、Xceptionに近い値を計測したことが驚きである。


モデルの精度比較

accuracy

Xception ≒ MobileNet > Inception > VGG16

loss

Xception < VGG16 < MobileNet < Inception

各エポックごとの学習速度

MobileNet > VGG16 > Inception > Xception

今後はAWSを使って、InceptionResNetV2モデルを検証してみたい。


モデル予測

最後に最も精度が高かったXceptionモデルで、testデータを予測。

モデル読み込み

model = load_model(model_dir + 'model_Xception-opt.hdf5', compile = False)

# 相互のインデックスを対応させながらシャッフル

def shuffle_samples(X, y):
zipped = list(zip(X, y))
np.random.shuffle(zipped)
X_result, y_result = zip(*zipped)
return np.asarray(X_result), np.asarray(y_result)
X_test, y_test = shuffle_samples(X_test, y_test)

testデータ30件の画像と正解ラベルを出力

# testデータ30件の正解ラベル

true_classes = np.argmax(y_test[0:30], axis = 1)

# testデータ30件の画像と正解ラベルを出力
plt.figure(figsize = (16, 6))
for i in range(30):
plt.subplot(3, 10, i + 1)
plt.axis("off")
plt.title(classes[true_classes[i]])
plt.imshow(X_test[i])
plt.show()

testデータ30件の画像と予測ラベル・予測確率を出力

# testデータ30件の予測ラベル

pred_classes = np.argmax(model.predict(X_test[0:30]), axis = 1)

# testデータ30件の予測確率
pred_probs = np.max(model.predict(X_test[0:30]), axis = 1)
pred_probs = ['{:.4f}'.format(i) for i in pred_probs]

# testデータ30件の画像と予測ラベル・予測確率を出力
plt.figure(figsize = (16, 6))
for i in range(30):
plt.subplot(3, 10, i + 1)
plt.axis("off")
if pred_classes[i] == true_classes[i]:
plt.title(classes[pred_classes[i]] + '\n' + pred_probs[i])
else:
plt.title(classes[pred_classes[i]] + '\n' + pred_probs[i], color = "red")
plt.imshow(X_test[i])
plt.show()

高い確率でクラス分類できているかなと。