Reference
AutoDIAL: Automatic DomaIn Alignment Layers
Data
images_train = mnist.train.images
labels_train = mnist.train.labels
images_test = mnist.test.images
labels_test = mnist.test.labels
# for train data
indices = np.random.choice(55000, 10000, replace = False)
images_train_0 = images_train[indices]
images_train_0_2d = np.reshape(images_train_0, (-1, 28, 28))
labels_train_0 = labels_train[indices]
images_train_flip_2d = images_train_0_2d[:, :, ::-1]
images_train_flip = np.reshape(images_train_0_2d[:, :, ::-1], (-1, 28*28))
labels_train_flip = labels_train[indices]
# images_train_30_2d = []
# for i in range(len(images_train_0)):
# images_train_30_2d.append(skimage.transform.rotate(images_train_0_2d[i], 30))
# images_train_30 = np.reshape(images_train_30_2d, (-1, 28*28))
# labels_train_30 = labels_train[indices]
# images_train_60_2d = []
# for i in range(len(images_train_0)):
# images_train_60_2d.append(skimage.transform.rotate(images_train_0_2d[i], 60))
# images_train_60 = np.reshape(images_train_60_2d, (-1, 28*28))
# labels_train_60 = labels_train[indices]
# images_train_90_2d = []
# for i in range(len(images_train_0)):
# images_train_90_2d.append(skimage.transform.rotate(images_train_0_2d[i], 90))
# images_train_90 = np.reshape(images_train_90_2d, (-1, 28*28))
# labels_train_90 = labels_train[indices]
# images_train_180_2d = []
# for i in range(len(images_train_0)):
# images_train_180_2d.append(skimage.transform.rotate(images_train_0_2d[i], 180))
# images_train_180 = np.reshape(images_train_180_2d, (-1, 28*28))
# labels_train_180 = labels_train[indices]
Sample Code: Target domain data are unlabeled.
# Domain Adaptation
class DA():
def __init__(self):
pass
def weight_variable(self, name, shape):
initializer = tf.truncated_normal_initializer(mean = 0.0, stddev = 0.01, dtype = tf.float32)
return tf.get_variable(name, shape, initializer = initializer)
def bias_variable(self, name, shape):
initializer = tf.constant_initializer(value = 0.0, dtype = tf.float32)
return tf.get_variable(name, shape, initializer = initializer)
def alpha_variable(self, name):
initializer = tf.constant_initializer(value = 0.75, dtype = tf.float32)
return tf.get_variable(name, shape = (), initializer = initializer)
def F_0(self, x, filter_size, n_filters_1, n_filters_2, n_units, keep_prob, reuse = False):
x_reshaped = tf.reshape(x, [-1, 28, 28, 1])
with tf.variable_scope('F_0', reuse = reuse):
w_1 = self.weight_variable('w_1', [filter_size, filter_size, 1, n_filters_1])
b_1 = self.bias_variable('b_1', [n_filters_1])
# conv
conv = tf.nn.conv2d(x_reshaped, w_1, strides = [1, 2, 2, 1], padding = 'SAME') + b_1
# batch norm
#batch_mean, batch_var = tf.nn.moments(conv, [0, 1, 2])
#conv = (conv - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
# relu
conv = tf.nn.relu(conv)
# max_pool
#conv = tf.nn.max_pool(conv, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1], padding = 'SAME')
w_2 = self.weight_variable('w_2', [filter_size, filter_size, n_filters_1, n_filters_2])
b_2 = self.bias_variable('b_2', [n_filters_2])
# conv
conv = tf.nn.conv2d(conv, w_2, strides = [1, 2, 2, 1], padding = 'SAME') + b_2
# batch norm
batch_mean, batch_var = tf.nn.moments(conv, [0, 1, 2])
conv = (conv - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
# relu
conv = tf.nn.relu(conv)
# max_pool
#conv = tf.nn.max_pool(conv, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1], padding = 'SAME')
conv_flat = tf.reshape(conv, [-1, 7 * 7 * n_filters_2])
w_3 = self.weight_variable('w_3', [7 * 7 * n_filters_2, n_units])
b_3 = self.bias_variable('b_3', [n_units])
fc = tf.nn.relu(tf.matmul(conv_flat, w_3) + b_3)
# batch norm
batch_mean, batch_var = tf.nn.moments(fc, [0])
fc = (fc - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
# dropout
#fc = tf.nn.dropout(fc, keep_prob)
feature = fc
return feature
def F_1(self, x, n_units_1, n_units_2, keep_prob, reuse = False):
with tf.variable_scope('F_1', reuse = reuse):
w_1 = self.weight_variable('w_1', [n_units_1, n_units_2])
b_1 = self.bias_variable('b_1', [n_units_2])
fc = tf.matmul(x, w_1) + b_1
# batch norm
batch_mean, batch_var = tf.nn.moments(fc, [0])
fc = (fc - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
# relu
fc = tf.nn.relu(fc)
# dropout
#fc = tf.nn.dropout(fc, keep_prob)
w_2 = self.weight_variable('w_2', [n_units_2, 10])
b_2 = self.bias_variable('b_2', [10])
fc = tf.matmul(fc, w_2) + b_2
logits = fc
return logits
def F_t(self, x, n_units_1, n_units_2, keep_prob, reuse = False):
with tf.variable_scope('F_t', reuse = reuse):
w_1 = self.weight_variable('w_1', [n_units_1, n_units_2])
b_1 = self.bias_variable('b_1', [n_units_2])
fc = tf.matmul(x, w_1) + b_1
# batch norm
batch_mean, batch_var = tf.nn.moments(fc, [0])
fc = (fc - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
# relu
fc = tf.nn.relu(fc)
# dropout
#fc = tf.nn.dropout(fc, keep_prob)
w_2 = self.weight_variable('w_2', [n_units_2, 10])
b_2 = self.bias_variable('b_2', [10])
fc = tf.matmul(fc, w_2) + b_2
logits = fc
return logits
def F_shared(self, x, n_units_1, n_units_2, keep_prob, reuse = False):
with tf.variable_scope('F_shared', reuse = reuse):
w_1 = self.weight_variable('w_1', [n_units_1, n_units_2])
b_1 = self.bias_variable('b_1', [n_units_2])
fc = tf.matmul(x, w_1) + b_1
# batch norm
batch_mean, batch_var = tf.nn.moments(fc, [0])
fc = (fc - batch_mean) / (tf.sqrt(batch_var) + 1e-10)
# relu
fc = tf.nn.relu(fc)
# dropout
#fc = tf.nn.dropout(fc, keep_prob)
w_2 = self.weight_variable('w_2', [n_units_2, 10])
b_2 = self.bias_variable('b_2', [10])
fc = tf.matmul(fc, w_2) + b_2
logits = fc
return logits
def F_DA(self, x_1, x_2, n_units_1, n_units_2, keep_prob, reuse = False):
with tf.variable_scope('DA', reuse = reuse):
w_1 = self.weight_variable('w_1', [n_units_1, n_units_2])
b_1 = self.bias_variable('b_1', [n_units_2])
fc_1 = tf.matmul(x_1, w_1) + b_1
fc_2 = tf.matmul(x_2, w_1) + b_1
# Full alignment
# batch norm
batch_mean_1, batch_var_1 = tf.nn.moments(fc_1, [0])
fc_1 = (fc_1 - batch_mean_1) / (tf.sqrt(batch_var_1) + 1e-10)
batch_mean_2, batch_var_2 = tf.nn.moments(fc_2, [0])
fc_2 = (fc_2 - batch_mean_2) / (tf.sqrt(batch_var_2) + 1e-10)
# Zero alignment
# batch norm
#batch_mean_1, batch_var_1 = tf.nn.moments(fc_1, [0])
#batch_mean_2, batch_var_2 = tf.nn.moments(fc_2, [0])
#fc_1 = 0.5 * (fc_1 - batch_mean_1) / (tf.sqrt(batch_var_1) + 1e-10) +\
# 0.5 * (fc_1 - batch_mean_2) / (tf.sqrt(batch_var_2) + 1e-10)
#fc_2 = 0.5 * (fc_2 - batch_mean_1) / (tf.sqrt(batch_var_1) + 1e-10) +\
# 0.5 * (fc_2 - batch_mean_2) / (tf.sqrt(batch_var_2) + 1e-10)
# relu
fc_1 = tf.nn.relu(fc_1)
fc_2 = tf.nn.relu(fc_2)
# dropout
#fc_1 = tf.nn.dropout(fc_1, keep_prob)
#fc_2 = tf.nn.dropout(fc_2, keep_prob)
w_2 = self.weight_variable('w_2', [n_units_2, 10])
b_2 = self.bias_variable('b_2', [10])
fc_1 = tf.matmul(fc_1, w_2) + b_2
fc_2 = tf.matmul(fc_2, w_2) + b_2
logits_1 = fc_1
logits_2 = fc_2
return logits_1, logits_2
def loss_cross_entropy(self, y, t):
cross_entropy = - tf.reduce_mean(tf.reduce_sum(t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), axis = 1))
return cross_entropy
def loss_entropy(self, p):
entropy = - tf.reduce_mean(tf.reduce_sum(p * tf.log(tf.clip_by_value(p, 1e-10, 1.0)), axis = 1))
return entropy
def loss_mutual_information(self, p):
p_ave = tf.reduce_mean(p, axis = 0)
h_y = -tf.reduce_sum(p_ave * tf.log(p_ave + 1e-16))
h_y_x = - tf.reduce_mean(tf.reduce_sum(p * tf.log(tf.clip_by_value(p, 1e-10, 1.0)), axis = 1))
mutual_info = h_y - h_y_x
return -mutual_info
def accuracy(self, y, t):
correct_preds = tf.equal(tf.argmax(y, axis = 1), tf.argmax(t, axis = 1))
accuracy = tf.reduce_mean(tf.cast(correct_preds, tf.float32))
return accuracy
def training(self, loss, learning_rate, var_list):
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
#optimizer = tf.train.GradientDescentOptimizer(learning_rate = learning_rate)
train_step = optimizer.minimize(loss, var_list = var_list)
return train_step
def training_clipped(self, loss, learning_rate, clip_norm, var_list):
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
grads_and_vars = optimizer.compute_gradients(loss, var_list = var_list)
clipped_grads_and_vars = [(tf.clip_by_norm(grad, clip_norm = clip_norm), \
var) for grad, var in grads_and_vars]
train_step = optimizer.apply_gradients(clipped_grads_and_vars)
return train_step
# train 0 and 1 first then t
def fit_01_t(self, images_train_1, labels_train_1, images_test_1, labels_test_1, \
images_train_t, labels_train_t, images_test_t, labels_test_t, \
filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, n_units_t, \
learning_rate, n_iter, batch_size, show_step, is_saving, model_path):
tf.reset_default_graph()
x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
feat = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = False)
logits_1 = self.F_1(feat, n_units_0, n_units_1, keep_prob, reuse = False)
probs_1 = tf.nn.softmax(logits_1)
loss_1 = self.loss_cross_entropy(probs_1, y)
logits_t = self.F_t(feat, n_units_0, n_units_t, keep_prob, reuse = False)
probs_t = tf.nn.softmax(logits_t)
loss_t = self.loss_cross_entropy(probs_t, y) # labeled data
#loss_t = self.loss_entropy(probs_t) # unlabeled data
var_list_0 = tf.trainable_variables('F_0')
var_list_1 = tf.trainable_variables('F_1')
var_list_t = tf.trainable_variables('F_t')
var_list_0_1 = var_list_0 + var_list_1
#var_list_0_t = var_list_0 + var_list_t
# Without Gradient Clipping
train_step_1 = self.training(loss_1, learning_rate, var_list_0_1)
train_step_t = self.training(loss_t, learning_rate, var_list_t)
# With Gradient Clipping
#train_step_1 = self.training_clipped(loss_1, learning_rate, 0.1, var_list_0_1)
#train_step_t = self.training_clipped(loss_t, learning_rate, 0.1, var_list_0_t)
acc_1 = self.accuracy(probs_1, y)
acc_t = self.accuracy(probs_t, y)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init)
# for 0 and 1
history_loss_train_1 = []
history_loss_test_1 = []
history_acc_train_1 = []
history_acc_test_1 = []
for i in range(n_iter):
# Train
rand_index = np.random.choice(len(images_train_1), size = batch_size)
x_batch = images_train_1[rand_index]
y_batch = labels_train_1[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 0.7}
sess.run(train_step_1, feed_dict = feed_dict)
temp_loss = sess.run(loss_1, feed_dict = feed_dict)
temp_acc = sess.run(acc_1, feed_dict = feed_dict)
history_loss_train_1.append(temp_loss)
history_acc_train_1.append(temp_acc)
if (i + 1) % show_step == 0:
print ('-' * 15)
print ('Iteration: ' + str(i + 1) + ' Loss_1: ' + str(temp_loss) + \
' Accuracy_1: ' + str(temp_acc))
# Test
rand_index = np.random.choice(len(images_test_1), size = batch_size)
x_batch = images_test_1[rand_index]
y_batch = labels_test_1[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 1.0}
temp_loss = sess.run(loss_1, feed_dict = feed_dict)
temp_acc = sess.run(acc_1, feed_dict = feed_dict)
history_loss_test_1.append(temp_loss)
history_acc_test_1.append(temp_acc)
if is_saving:
model_path = saver.save(sess, model_path)
print ('done saving at ', model_path)
print ('-'* 15)
fig = plt.figure(figsize = (10, 3))
ax1 = fig.add_subplot(1, 2, 1)
ax1.plot(range(n_iter), history_loss_train_1, 'b-', label = 'Train')
ax1.plot(range(n_iter), history_loss_test_1, 'r--', label = 'Test')
ax1.set_title('Loss_1')
ax1.legend(loc = 'upper right')
ax2 = fig.add_subplot(1, 2, 2)
ax2.plot(range(n_iter), history_acc_train_1, 'b-', label = 'Train')
ax2.plot(range(n_iter), history_acc_test_1, 'r--', label = 'Test')
ax2.set_ylim(0.0, 1.0)
ax2.set_title('Accuracy_1')
ax2.legend(loc = 'lower right')
plt.show()
# for t
history_loss_train_t = []
history_loss_test_t = []
history_acc_train_t = []
history_acc_test_t = []
for i in range(n_iter):
# Train
rand_index = np.random.choice(len(images_train_t), size = batch_size)
x_batch = images_train_t[rand_index]
y_batch = labels_train_t[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 0.7}
sess.run(train_step_t, feed_dict = feed_dict)
temp_loss = sess.run(loss_t, feed_dict = feed_dict)
temp_acc = sess.run(acc_t, feed_dict = feed_dict)
history_loss_train_t.append(temp_loss)
history_acc_train_t.append(temp_acc)
if (i + 1) % show_step == 0:
print ('-' * 15)
print ('Iteration: ' + str(i + 1) + ' Loss_t: ' + str(temp_loss) + \
' Accuracy_t: ' + str(temp_acc))
# Test
rand_index = np.random.choice(len(images_test_t), size = batch_size)
x_batch = images_test_t[rand_index]
y_batch = labels_test_t[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 1.0}
temp_loss = sess.run(loss_t, feed_dict = feed_dict)
temp_acc = sess.run(acc_t, feed_dict = feed_dict)
history_loss_test_t.append(temp_loss)
history_acc_test_t.append(temp_acc)
if is_saving:
model_path = saver.save(sess, model_path)
print ('done saving at ', model_path)
print ('-'* 15)
fig = plt.figure(figsize = (10, 3))
ax1 = fig.add_subplot(1, 2, 1)
ax1.plot(range(n_iter), history_loss_train_t, 'b-', label = 'Train')
ax1.plot(range(n_iter), history_loss_test_t, 'r--', label = 'Test')
ax1.set_title('Loss_t')
ax1.legend(loc = 'upper right')
ax2 = fig.add_subplot(1, 2, 2)
ax2.plot(range(n_iter), history_acc_train_t, 'b-', label = 'Train')
ax2.plot(range(n_iter), history_acc_test_t, 'r--', label = 'Test')
ax2.set_ylim(0.0, 1.0)
ax2.set_title('Accuracy_t')
ax2.legend(loc = 'lower right')
plt.show()
# train 0, 1 and t alternately based on different F's or a shared F
def fit_01t(self, images_train_1, labels_train_1, images_test_1, labels_test_1, \
images_train_t, labels_train_t, images_test_t, labels_test_t, \
filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, n_units_t, \
learning_rate, n_iter, batch_size, show_step, is_saving, model_path):
tf.reset_default_graph()
x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
feat = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = False)
#logits_1 = self.F_1(feat, n_units_0, n_units_1, keep_prob, reuse = False)
logits_1 = self.F_shared(feat, n_units_0, n_units_1, keep_prob, reuse = False)
probs_1 = tf.nn.softmax(logits_1)
loss_1 = self.loss_cross_entropy(probs_1, y)
#logits_t = self.F_t(feat, n_units_0, n_units_t, keep_prob, reuse = False)
logits_t = self.F_shared(feat, n_units_0, n_units_t, keep_prob, reuse = True)
probs_t = tf.nn.softmax(logits_t)
loss_t = self.loss_cross_entropy(probs_t, y) # labeled data
#loss_t = self.loss_entropy(probs_t) # unlabeled data
var_list_0 = tf.trainable_variables('F_0')
var_list_1 = tf.trainable_variables('F_1')
var_list_t = tf.trainable_variables('F_t')
var_list_0_1 = var_list_0 + var_list_1
var_list_0_t = var_list_0 + var_list_t
#var_list_all = var_list_0 + var_list_1 + var_list_t
# Without Gradient Clipping
train_step_1 = self.training(loss_1, learning_rate, var_list_0_1)
train_step_t = self.training(loss_t, learning_rate, var_list_0_t)
# With Gradient Clipping
#train_step_1 = self.training_clipped(loss_1, learning_rate, 0.1, var_list_0_1)
#train_step_t = self.training_clipped(loss_t, learning_rate, 0.1, var_list_0_t)
acc_1 = self.accuracy(probs_1, y)
acc_t = self.accuracy(probs_t, y)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init)
history_loss_train_1 = []
history_loss_test_1 = []
history_acc_train_1 = []
history_acc_test_1 = []
history_loss_train_t = []
history_loss_test_t = []
history_acc_train_t = []
history_acc_test_t = []
for i in range(n_iter):
# for 0 and 1
# Train
rand_index = np.random.choice(len(images_train_1), size = batch_size)
x_batch = images_train_1[rand_index]
y_batch = labels_train_1[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 0.7}
sess.run(train_step_1, feed_dict = feed_dict)
temp_loss = sess.run(loss_1, feed_dict = feed_dict)
temp_acc = sess.run(acc_1, feed_dict = feed_dict)
history_loss_train_1.append(temp_loss)
history_acc_train_1.append(temp_acc)
if (i + 1) % show_step == 0:
print ('-' * 15)
print ('Iteration: ' + str(i + 1) + ' Loss_1: ' + str(temp_loss) + \
' Accuracy_1: ' + str(temp_acc))
# Test
rand_index = np.random.choice(len(images_test_1), size = batch_size)
x_batch = images_test_1[rand_index]
y_batch = labels_test_1[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 1.0}
temp_loss = sess.run(loss_1, feed_dict = feed_dict)
temp_acc = sess.run(acc_1, feed_dict = feed_dict)
history_loss_test_1.append(temp_loss)
history_acc_test_1.append(temp_acc)
# for 0 and t
# Train
rand_index = np.random.choice(len(images_train_t), size = batch_size)
x_batch = images_train_t[rand_index]
y_batch = labels_train_t[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 0.7}
sess.run(train_step_t, feed_dict = feed_dict)
temp_loss = sess.run(loss_t, feed_dict = feed_dict)
temp_acc = sess.run(acc_t, feed_dict = feed_dict)
history_loss_train_t.append(temp_loss)
history_acc_train_t.append(temp_acc)
if (i + 1) % show_step == 0:
print ('-' * 15)
print ('Iteration: ' + str(i + 1) + ' Loss_t: ' + str(temp_loss) + \
' Accuracy_t: ' + str(temp_acc))
# Test
rand_index = np.random.choice(len(images_test_t), size = batch_size)
x_batch = images_test_t[rand_index]
y_batch = labels_test_t[rand_index]
feed_dict = {x: x_batch, y: y_batch, keep_prob: 1.0}
temp_loss = sess.run(loss_t, feed_dict = feed_dict)
temp_acc = sess.run(acc_t, feed_dict = feed_dict)
history_loss_test_t.append(temp_loss)
history_acc_test_t.append(temp_acc)
if is_saving:
model_path = saver.save(sess, model_path)
print ('done saving at ', model_path)
print ('-'* 15)
fig = plt.figure(figsize = (10, 7))
ax1 = fig.add_subplot(2, 2, 1)
ax1.plot(range(n_iter), history_loss_train_1, 'b-', label = 'Train')
ax1.plot(range(n_iter), history_loss_test_1, 'r--', label = 'Test')
ax1.set_title('Loss_1')
ax1.legend(loc = 'upper right')
ax2 = fig.add_subplot(2, 2, 2)
ax2.plot(range(n_iter), history_acc_train_1, 'b-', label = 'Train')
ax2.plot(range(n_iter), history_acc_test_1, 'r--', label = 'Test')
ax2.set_ylim(0.0, 1.0)
ax2.set_title('Accuracy_1')
ax2.legend(loc = 'lower right')
ax1 = fig.add_subplot(2, 2, 3)
ax1.plot(range(n_iter), history_loss_train_t, 'b-', label = 'Train')
ax1.plot(range(n_iter), history_loss_test_t, 'r--', label = 'Test')
ax1.set_title('Loss_t')
ax1.legend(loc = 'upper right')
ax2 = fig.add_subplot(2, 2, 4)
ax2.plot(range(n_iter), history_acc_train_t, 'b-', label = 'Train')
ax2.plot(range(n_iter), history_acc_test_t, 'r--', label = 'Test')
ax2.set_ylim(0.0, 1.0)
ax2.set_title('Accuracy_t')
ax2.legend(loc = 'lower right')
plt.show()
# train 0, 1 and t simultaneously with DA
def fit_da(self, images_train_1, labels_train_1, images_test_1, labels_test_1, \
images_train_t, labels_train_t, images_test_t, labels_test_t, \
filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, n_units_t, \
learning_rate, n_iter, batch_size, show_step, is_saving, model_path):
tf.reset_default_graph()
x_1 = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
x_t = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
y_1 = tf.placeholder(shape = [None, 10], dtype = tf.float32)
y_t = tf.placeholder(shape = [None, 10], dtype = tf.float32)
keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
feat_1 = self.F_0(x_1, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = False)
feat_t = self.F_0(x_t, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = True)
logits_1, logits_t = self.F_DA(feat_1, feat_t, n_units_0, n_units_1, keep_prob, reuse = False)
probs_1 = tf.nn.softmax(logits_1)
loss_1 = self.loss_cross_entropy(probs_1, y_1)
probs_t = tf.nn.softmax(logits_t)
#loss_t = self.loss_cross_entropy(probs_t, y_t) # labeled data
loss_t = self.loss_entropy(probs_t) # unlabeled data
loss_total = loss_1 + loss_t
var_list_0 = tf.trainable_variables('F_0')
var_list_1 = tf.trainable_variables('F_1')
var_list_t = tf.trainable_variables('F_t')
var_list_all = var_list_0 + var_list_1 + var_list_t
# Without Gradient Clipping
train_step = self.training(loss_total, learning_rate, var_list_all)
# With Gradient Clipping
#train_step = self.training_clipped(loss_total, learning_rate, 0.1, var_list_all)
acc_1 = self.accuracy(probs_1, y_1)
acc_t = self.accuracy(probs_t, y_t)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init)
history_loss_train = []
history_loss_test = []
history_acc_train_1 = []
history_acc_test_1 = []
history_acc_train_t = []
history_acc_test_t = []
for i in range(n_iter):
# Train
rand_index = np.random.choice(len(images_train_1), size = batch_size)
x_batch_1 = images_train_1[rand_index]
y_batch_1 = labels_train_1[rand_index]
rand_index = np.random.choice(len(images_train_t), size = batch_size)
x_batch_t = images_train_t[rand_index]
y_batch_t = labels_train_t[rand_index]
feed_dict = {x_1: x_batch_1, x_t: x_batch_t, y_1: y_batch_1, y_t: y_batch_t, keep_prob: 0.7}
sess.run(train_step, feed_dict = feed_dict)
temp_loss = sess.run(loss_total, feed_dict = feed_dict)
temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
temp_acc_t = sess.run(acc_t, feed_dict = feed_dict)
history_loss_train.append(temp_loss)
history_acc_train_1.append(temp_acc_1)
history_acc_train_t.append(temp_acc_t)
if (i + 1) % show_step == 0:
print ('-' * 15)
print ('Iteration: ' + str(i + 1) + ' Loss: ' + str(temp_loss) +\
' Accuracy_1: ' + str(temp_acc_1) +\
' Accuracy_t: ' + str(temp_acc_t))
# Test
rand_index = np.random.choice(len(images_test_1), size = batch_size)
x_batch_1 = images_test_1[rand_index]
y_batch_1 = labels_test_1[rand_index]
rand_index = np.random.choice(len(images_test_t), size = batch_size)
x_batch_t = images_test_t[rand_index]
y_batch_t = labels_test_t[rand_index]
feed_dict = {x_1: x_batch_1, x_t: x_batch_t, y_1: y_batch_1, y_t: y_batch_t, keep_prob: 0.7}
temp_loss = sess.run(loss_total, feed_dict = feed_dict)
temp_acc_1 = sess.run(acc_1, feed_dict = feed_dict)
temp_acc_t = sess.run(acc_t, feed_dict = feed_dict)
history_loss_test.append(temp_loss)
history_acc_test_1.append(temp_acc_1)
history_acc_test_t.append(temp_acc_t)
if is_saving:
model_path = saver.save(sess, model_path)
print ('done saving at ', model_path)
print ('-'* 15)
fig = plt.figure(figsize = (10, 7))
ax1 = fig.add_subplot(2, 2, 1)
ax1.plot(range(n_iter), history_loss_train, 'b-', label = 'Train')
ax1.plot(range(n_iter), history_loss_test, 'r--', label = 'Test')
ax1.set_title('Loss')
ax1.legend(loc = 'upper right')
ax2 = fig.add_subplot(2, 2, 3)
ax2.plot(range(n_iter), history_acc_train_1, 'b-', label = 'Train')
ax2.plot(range(n_iter), history_acc_test_1, 'r--', label = 'Test')
ax2.set_ylim(0.0, 1.0)
ax2.set_title('Accuracy_1')
ax2.legend(loc = 'lower right')
ax2 = fig.add_subplot(2, 2, 4)
ax2.plot(range(n_iter), history_acc_train_t, 'b-', label = 'Train')
ax2.plot(range(n_iter), history_acc_test_t, 'r--', label = 'Test')
ax2.set_ylim(0.0, 1.0)
ax2.set_title('Accuracy_t')
ax2.legend(loc = 'lower right')
plt.show()
def predict_1(self, images, labels, filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, model_path):
x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
feat = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = True)
logits = self.F_1(feat, n_units_0, n_units_1, keep_prob, reuse = True)
probs = tf.nn.softmax(logits)
acc = self.accuracy(probs, y)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_path)
feed_dict = {x: images, y: labels, keep_prob: 1.0}
print('Accuracy_1: {:.4f}'.format(sess.run(acc, feed_dict = feed_dict)))
return sess.run(probs, feed_dict = feed_dict)
def predict_2(self, images, labels, filter_size, n_filters_1, n_filters_2, n_units_0, n_units_2, model_path):
x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
feat = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = True)
logits = self.F_2(feat, n_units_0, n_units_2, keep_prob, reuse = True)
probs = tf.nn.softmax(logits)
acc = self.accuracy(probs, y)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_path)
feed_dict = {x: images, y: labels, keep_prob: 1.0}
print('Accuracy_2: {:.4f}'.format(sess.run(acc, feed_dict = feed_dict)))
return sess.run(probs, feed_dict = feed_dict)
def predict_t(self, images, labels, filter_size, n_filters_1, n_filters_2, n_units_0, n_units_t, model_path):
x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
feat = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = True)
logits = self.F_t(feat, n_units_0, n_units_t, keep_prob, reuse = True)
probs = tf.nn.softmax(logits)
acc = self.accuracy(probs, y)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_path)
feed_dict = {x: images, y: labels, keep_prob: 1.0}
print('Accuracy_t: {:.4f}'.format(sess.run(acc, feed_dict = feed_dict)))
return sess.run(probs, feed_dict = feed_dict)
def predict_da(self, images, labels, filter_size, n_filters_1, n_filters_2, n_units_0, n_units_t, model_path):
x = tf.placeholder(shape = [None, 28 * 28], dtype = tf.float32)
y = tf.placeholder(shape = [None, 10], dtype = tf.float32)
keep_prob = tf.placeholder(shape = (), dtype = tf.float32)
feat_1 = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = True)
feat_t = self.F_0(x, filter_size, n_filters_1, n_filters_2, n_units_0, \
keep_prob, reuse = True)
logits_1, logits_t = self.F_DA(feat_1, feat_t, n_units_0, n_units_1, keep_prob, reuse = True)
probs_1 = tf.nn.softmax(logits_1)
acc_1 = self.accuracy(probs_1, y)
probs_t = tf.nn.softmax(logits_t)
acc_t = self.accuracy(probs_t, y)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_path)
feed_dict = {x: images, y: labels, keep_prob: 1.0}
print('Accuracy_1: {:.4f}'.format(sess.run(acc_1, feed_dict = feed_dict)))
print('Accuracy_t: {:.4f}'.format(sess.run(acc_t, feed_dict = feed_dict)))
return sess.run([probs_1, probs_t], feed_dict = feed_dict)
Parameters
filter_size = 3
n_filters_1 = 16
n_filters_2 = 16
n_fc_0 = 128
n_fc_1 = 64
n_fc_2 = 64
learning_rate = 0.01
n_iter = 300
batch_size = 64
show_step = 100
model_path = 'datalab/model'
Output results 1
images_train_1 = images_train_0
labels_train_1 = labels_train_0
images_test_1 = images_test_30
labels_test_1 = labels_test_30
images_train_t = images_train_30
labels_train_t = labels_train_30
images_test_t = images_test_30
labels_test_t = labels_test_30
da.fit_da(images_train_1, labels_train_1, images_test_1, labels_test_1, \
images_train_t, labels_train_t, images_test_t, labels_test_t, \
filter_size, n_filters_1, n_filters_2, n_units_0, n_units_1, n_units_t, \
learning_rate, n_iter, batch_size, show_step, is_saving, model_path)
