LoginSignup
0
0

More than 5 years have passed since last update.

@takeshikondo(Takeshi Kondo)

Structured Output Prediction (Baseline NN ) の実装に関するメモ

Posted at

ポイント

  • Baseline Neural Network を実装し、具体的な数値で確認。

レファレンス

1. Learning Structured Output Representation using Deep Conditional Generative Models

サンプルコード

class BaseLine():

  def __init__(self):
    pass

  def weight_variable(self, name, shape):
    initializer = tf.truncated_normal_initializer(mean = 0.0, stddev = 0.01, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)

  def bias_variable(self, name, shape):
    initializer = tf.constant_initializer(value = 0.0, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)

  def const_zero(self, name, shape):
    initializer = tf.constant_initializer(value = 0.0, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)

  def const_one(self, name, shape):
    initializer = tf.constant_initializer(value = 1.0, dtype = tf.float32)
    return tf.get_variable(name, shape, initializer = initializer)

  def inference(self, x, n_in, n_units_1, n_units_2, n_out):
    with tf.variable_scope('mlp'):
      w_1 = self.weight_variable('w_1', [n_in, n_units_1])
      b_1 = self.bias_variable('b_1', [n_units_1])

      h_1 = tf.add(tf.matmul(x, w_1), b_1)

      # batch normalization
      beta_1 = self.const_zero('beta_1', [n_units_1])
      gamma_1 = self.const_one('gamma_1', [n_units_1])

      mean_1, var_1 = tf.nn.moments(h_1, [0])
      h_1 = gamma_1 * (h_1 - mean_1) / tf.sqrt(var_1 + 1e-5) + beta_1

      bn_1 = tf.nn.sigmoid(h_1)

      w_2 = self.weight_variable('w_2', [n_units_1, n_units_2])
      b_2 = self.bias_variable('b_2', [n_units_2])

      h_2 = tf.add(tf.matmul(bn_1, w_2), b_2)

      # batch normalization
      beta_2 = self.const_zero('beta_2', [n_units_2])
      gamma_2 = self.const_one('gamma_2', [n_units_2])

      mean_2, var_2 = tf.nn.moments(h_2, [0])
      h_2 = gamma_2 * (h_2 - mean_2) / tf.sqrt(var_2 + 1e-5) + beta_2

      bn_2 = tf.nn.sigmoid(h_2)

      w_3 = self.weight_variable('w_3', [n_units_2, n_out])
      b_3 = self.bias_variable('b_3', [n_out])

      h_3 = tf.add(tf.matmul(bn_2, w_3), b_3)

      return tf.nn.sigmoid(h_3)

  def loss(self, y, t):
    return - tf.reduce_mean(tf.reduce_sum(t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)) + \
                                                             (1 - t) * tf.log(tf.clip_by_value(1 - y, 1e-10, 1.0)), axis = 1))
    #return tf.reduce_mean(tf.square(t - y))

  def training(self, loss, learning_rate):
    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
    train_step = optimizer.minimize(loss)
    return train_step

  def training_clipped(self, loss, learning_rate, clip_norm):
    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)

    grads_and_vars = optimizer.compute_gradients(loss)
    clipped_grads_and_vars = [(tf.clip_by_norm(grad, clip_norm = clip_norm), \
                             var) for grad, var in grads_and_vars]
    train_step = optimizer.apply_gradients(clipped_grads_and_vars)

    return train_step

  def fit(self, images_train_x, images_train_y, labels_train, images_test_x, images_test_y, labels_test, \
          n_in, n_units_1, n_units_2, n_out, \
          learning_rate, n_iter, batch_size, show_step, is_saving, model_path):

    tf.reset_default_graph()

    x = tf.placeholder(shape = [None, 14 * 28], dtype = tf.float32)
    t = tf.placeholder(shape = [None, 14 * 28], dtype = tf.float32)

    y = self.inference(x, n_in, n_units_1, n_units_2, n_out)
    loss = self.loss(y, t)
    train_step = self.training(loss, learning_rate)

    init = tf.global_variables_initializer()
    saver = tf.train.Saver()

    with tf.Session() as sess:

      sess.run(init)

      history_loss_train = []
      history_loss_test = []

      for i in range(n_iter):
        # Train
        rand_index = np.random.choice(len(images_train_x), size = batch_size)
        x_batch = images_train_x[rand_index]
        y_batch = images_train_y[rand_index]

        feed_dict = {x: x_batch, t: y_batch}

        sess.run(train_step, feed_dict = feed_dict)

        temp_loss = sess.run(loss, feed_dict = feed_dict)
        history_loss_train.append(temp_loss)

        if (i + 1) % show_step == 0:
          print ('--------------------')
          print ('Iteration: ' + str(i + 1) + '  Loss: ' + str(temp_loss))

        # Test
        rand_index = np.random.choice(len(images_test_x), size = batch_size)
        x_batch = images_test_x[rand_index]
        y_batch = images_test_y[rand_index]

        feed_dict = {x: x_batch, t: y_batch}

        temp_loss = sess.run(loss, feed_dict = feed_dict)
        history_loss_test.append(temp_loss)

      if is_saving:
        model_path = saver.save(sess, model_path)
        print ('--------------------')
        print ('done saving at ', model_path)

    fig = plt.figure(figsize = (10, 3))
    ax1 = fig.add_subplot(1, 2, 1)
    ax1.plot(range(n_iter), history_loss_train, 'b-', label = 'Train')
    ax1.plot(range(n_iter), history_loss_test, 'r--', label = 'Test')
    ax1.set_title('Loss')
    ax1.legend(loc = 'upper right')

    plt.show()    

  def prediction(self, x, n_in, n_units_1, n_units_2, n_out, model_path):
    with tf.variable_scope('mlp', reuse = True):
      w_1 = self.weight_variable('w_1', [n_in, n_units_1])
      b_1 = self.bias_variable('b_1', [n_units_1])

      h_1 = tf.add(tf.matmul(x, w_1), b_1)

      # batch normalization
      beta_1 = self.const_zero('beta_1', [n_units_1])
      gamma_1 = self.const_one('gamma_1', [n_units_1])

      mean_1, var_1 = tf.nn.moments(h_1, [0])
      h_1 = gamma_1 * (h_1 - mean_1) / tf.sqrt(var_1 + 1e-5) + beta_1

      bn_1 = tf.nn.sigmoid(h_1)

      w_2 = self.weight_variable('w_2', [n_units_1, n_units_2])
      b_2 = self.bias_variable('b_2', [n_units_2])

      h_2 = tf.add(tf.matmul(bn_1, w_2), b_2)

      # batch normalization
      beta_2 = self.const_zero('beta_2', [n_units_2])
      gamma_2 = self.const_one('gamma_2', [n_units_2])

      mean_2, var_2 = tf.nn.moments(h_2, [0])
      h_2 = gamma_2 * (h_2 - mean_2) / tf.sqrt(var_2 + 1e-5) + beta_2

      bn_2 = tf.nn.sigmoid(h_2)

      w_3 = self.weight_variable('w_3', [n_units_2, n_out])
      b_3 = self.bias_variable('b_3', [n_out])

      h_3 = tf.add(tf.matmul(bn_2, w_3), b_3)

      y = tf.nn.sigmoid(h_3)

    saver = tf.train.Saver()

    with tf.Session() as sess:

      saver.restore(sess, model_path)

      return sess.run(y)

データ

images_train_x = mnist.train.images[:, :14*28]
images_train_y = mnist.train.images[:, 14*28:]
labels_train = mnist.train.labels
images_test_x = mnist.test.images[:, :14*28]
images_test_y = mnist.test.images[:, 14*28:]
labels_test = mnist.test.labels

パラメータ

n_in = 14*28
n_units_1 = 64
n_units_2 = 64
n_out = 14*28
learning_rate = 0.01
batch_size = 100

アウトプット

image.png

index = np.random.choice(10000, batch_size)
x = images_test_x[index]
labels = np.argmax(labels_test[index], axis = 1)

predicts = bl.prediction(x, n_in, n_units_1, n_units_2, n_out, model_path)

print (np.shape(x))
print (np.shape(predicts))

fig = plt.figure(figsize = (4, 1))

ax1 = fig.add_subplot(2, 3, 1)
ax1.imshow(np.reshape(x[0], [14, 28]), cmap = 'gray')
ax1.set_axis_off()
#ax1.set_title('Input')

ax2 = fig.add_subplot(2, 3, 4)
ax2.imshow(np.reshape(predicts[0], [14, 28]), cmap = 'gray')
ax2.set_axis_off()
#ax2.set_title('Predict')

ax3 = fig.add_subplot(2, 3, 2)
ax3.imshow(np.reshape(x[1], [14, 28]), cmap = 'gray')
ax3.set_axis_off()
#ax3.set_title('Input')

ax4 = fig.add_subplot(2, 3, 5)
ax4.imshow(np.reshape(predicts[1], [14, 28]), cmap = 'gray')
ax4.set_axis_off()
#ax4.set_title('Predict')

ax5 = fig.add_subplot(2, 3, 3)
ax5.imshow(np.reshape(x[2], [14, 28]), cmap = 'gray')
ax5.set_axis_off()
#ax5.set_title('Input')

ax6 = fig.add_subplot(2, 3, 6)
ax6.imshow(np.reshape(predicts[2], [14, 28]), cmap = 'gray')
ax6.set_axis_off()
#ax6.set_title('Predict')

plt.show()

image.png

0
0
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
0
0