Gradient Clipping の実装に関するメモ

ポイント

• シンプルな数値計算で確認。

サンプルコード

  def loss(self, x, init_value):
    with tf.variable_scope('clip'):
      init_const = tf.constant_initializer(value = \
               init_value, dtype = tf.float32)
      w = tf.get_variable('w', shape = (), \
               initializer = init_const)
      var_list = tf.trainable_variables()
    return (w - x) ** 2, var_list

  def training(self, loss, var_list, learning_rate):
    optimizer = \
          tf.train.GradientDescentOptimizer( \ 
          learning_rate = learning_rate)

    # 1 using minimize
    #train_step = optimizer.minimize(loss)

    # 2 not using minimize 
    #grads_and_vars = optimizer.compute_gradients( \
    #         loss, var_list)
    #train_step = \
    #         optimizer.apply_gradients(grads_and_vars)

    # 3 gradient clipping
    grads_and_vars = optimizer.compute_gradients(loss, \
             var_list)
    clipped_grads_and_vars = [(tf.clip_by_value(grad, \
            clip_value_min = -3.0, clip_value_max = \
            3.0), var) for grad, var in grads_and_vars]
    train_step = \
             optimizer.apply_gradients( \ 
             clipped_grads_and_vars)

    return train_step, grads_and_vars, \
                     clipped_grads_and_vars

  def fit(self, x, init_value, learning_rate):

    tf.reset_default_graph()

    loss, var_list = self.loss(x, init_value)
    train_step, grads_and_vars, clipped_grads_and_vars \
       = self.training(loss, var_list, learning_rate)

    init = tf.global_variables_initializer()

    with tf.Session() as sess:
      sess.run(init)

      for i in range(5):
        temp_var = sess.run(grads_and_vars[0][1])
        temp_grad = sess.run(grads_and_vars[0][0])
        temp_clipped_var = \
              sess.run(clipped_grads_and_vars[0][1])
        temp_clipped_grad = \
              sess.run(clipped_grads_and_vars[0][0])
        temp_loss = sess.run(loss)

        print ('var: {0:.3f},  grad: {1:.3f}, \
           loss: {2:.3f}'.format(temp_clipped_var, \
           temp_clipped_grad, temp_loss))
        sess.run(train_step)

数値計算例

入力：
x = 2.0
init_value = 0.0
learning_rate = 0.1

出力：
var: 0.000, grad: -3.000, loss: 4.000
var: 0.300, grad: -3.000, loss: 2.890
var: 0.600, grad: -2.800, loss: 1.960
var: 0.880, grad: -2.240, loss: 1.254
var: 1.104, grad: -1.792, loss: 0.803