やりたいこと

tensorflow keras で各epochにかかった時間を計測して、どこかに(今回はmlflow)へ保存する。

方法

Callbackを自作して対応。

コード

import mlflow
import time
import tensorflow as tf

class SaveElapsedTime(tf.keras.callbacks.Callback):
    def __init__(self):
        self.epoch_start_time = None
        self.train_start_time = None

    def _cal_elapsed_time(self, st):
        return = time.time() - st

    def on_train_begin(self, logs=None):
        self.train_start = time.time()

    def on_train_end(self, logs=None):
        et = self._cal_elapsed_time(self.train_start_time)
        mlflow.log_param('total_train_elapsed_time', et)

    def on_epoch_begin(self, epoch, logs=None):
        self.epoch_start_time = time.time()

    def on_epoch_end(self, epoch, logs=None):
        et = self._cal_elapsed_time(self.epoch_start_time)
        mlflow.log_metric('epoch_elapsed_time', et, step=epoch+1)
        print('{} epoch fin. {:.2f} [sec]'.format(epoch+1, elapsed_time), file=sys.stderr, flush=True)