動作環境

GeForce GTX 1070 (8GB)
ASRock Z170M Pro4S [Intel Z170chipset]
Ubuntu 16.04 LTS desktop amd64
TensorFlow v1.1.0
cuDNN v5.1 for Linux
CUDA v8.0
Python 3.5.2
IPython 6.0.0 -- An enhanced Interactive Python.
gcc (Ubuntu 5.4.0-6ubuntu1~16.04.4) 5.4.0 20160609
GNU bash, version 4.3.48(1)-release (x86_64-pc-linux-gnu)

概要

This article is related to ADDA (light scattering simulator based on the discrete dipole approximation).

bash > 複数ディレクトリ内のファイルのTFRerocds化 (個別処理)
において作成した複数のTFRecordsファイルを１つのTFRecordsファイルにまとめる。

複数ファイルをそのままTensorFlowで読込んで学習させる方法もありそうだが、今回は１つのファイルにしてみる。

ファイル構成

$ ls ../run3*/IntField-Y*tfrecords
../run353_sphere_g26_m1.33/IntField-Y_170709.tfrecords  ../run361_sphere_g26_m1.4/IntField-Y_170709.tfrecords   ../run369_sphere_g26_m1.45/IntField-Y_170709.tfrecords
../run354_sphere_g26_m1.33/IntField-Y_170709.tfrecords  ../run362_sphere_g26_m1.4/IntField-Y_170709.tfrecords   ../run370_sphere_g26_m1.45/IntField-Y_170709.tfrecords
../run355_sphere_g26_m1.33/IntField-Y_170709.tfrecords  ../run363_sphere_g26_m1.4/IntField-Y_170709.tfrecords   ../run371_sphere_g26_m1.5/IntField-Y_170709.tfrecords
../run356_sphere_g26_m1.33/IntField-Y_170709.tfrecords  ../run364_sphere_g26_m1.4/IntField-Y_170709.tfrecords   ../run372_sphere_g26_m1.5/IntField-Y_170709.tfrecords
../run357_sphere_g26_m1.33/IntField-Y_170709.tfrecords  ../run365_sphere_g26_m1.45/IntField-Y_170709.tfrecords  ../run373_sphere_g26_m1.5/IntField-Y_170709.tfrecords
../run358_sphere_g26_m1.33/IntField-Y_170709.tfrecords  ../run366_sphere_g26_m1.45/IntField-Y_170709.tfrecords  ../run374_sphere_g26_m1.5/IntField-Y_170709.tfrecords
../run359_sphere_g26_m1.4/IntField-Y_170709.tfrecords   ../run367_sphere_g26_m1.45/IntField-Y_170709.tfrecords  ../run375_sphere_g26_m1.5/IntField-Y_170709.tfrecords
../run360_sphere_g26_m1.4/IntField-Y_170709.tfrecords   ../run368_sphere_g26_m1.45/IntField-Y_170709.tfrecords  ../run376_sphere_g26_m1.5/IntField-Y_170709.tfrecords

code v0.2

combine_TFRecords_170722.py

import numpy as np
import tensorflow as tf
import glob
import sys

"""
v0.2 Jul. 22, 2017
    - add combineTFRecords()
    - add _bytes_feature()
    - add _int64_feature()
    - add convert_to_raw()
    - add get_feature_float32()
v0.1 Jul. 22, 2017
    - get file list
"""

# on
#   Ubuntu 16.04 LTS
#   TensorFlow v1.1
#   Python 3.5.2

# codingrule:PEP8


OUT_FILE = 'combined_IntField-Y_170722.tfrecords'


def get_feature_float32(example, feature_name):
    wrk_raw = (example.features.feature[feature_name]
               .bytes_list
               .value[0])
    wrk_1d = np.fromstring(wrk_raw, dtype=np.float32)
    wrk_org = wrk_1d.reshape([1, -1])
    return wrk_org


def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))


def convert_to_raw(orgval):
    wrk_org = np.array(orgval, dtype=np.float32)
    wrk_raw = wrk_org.tostring()
    return wrk_raw


def combineTFRecords(inputfile, tfwriter):
    record_iterator = tf.python_io.tf_record_iterator(path=inputfile)
    for record in record_iterator:
        # --- 1. read
        example = tf.train.Example()
        example.ParseFromString(record)

        xpos_org = get_feature_float32(example, 'xpos_raw')
        ypos_org = get_feature_float32(example, 'ypos_raw')
        zpos_org = get_feature_float32(example, 'zpos_raw')
        mr_org = get_feature_float32(example, 'mr_raw')
        mi_org = get_feature_float32(example, 'mi_raw')
        exr_org = get_feature_float32(example, 'exr_raw')
        exi_org = get_feature_float32(example, 'exi_raw')
        eyr_org = get_feature_float32(example, 'eyr_raw')
        eyi_org = get_feature_float32(example, 'eyi_raw')
        ezr_org = get_feature_float32(example, 'ezr_raw')
        ezi_org = get_feature_float32(example, 'ezi_raw')

        # --- 2. output
        example = tf.train.Example(features=tf.train.Features(feature={
            'xpos_raw': _bytes_feature(convert_to_raw(xpos_org)),
            'ypos_raw': _bytes_feature(convert_to_raw(ypos_org)),
            'zpos_raw': _bytes_feature(convert_to_raw(zpos_org)),
            'mr_raw': _bytes_feature(convert_to_raw(mr_org)),
            'mi_raw': _bytes_feature(convert_to_raw(mi_org)),
            'exr_raw': _bytes_feature(convert_to_raw(exr_org)),
            'exi_raw': _bytes_feature(convert_to_raw(exi_org)),
            'eyr_raw': _bytes_feature(convert_to_raw(eyr_org)),
            'eyi_raw': _bytes_feature(convert_to_raw(eyi_org)),
            'ezr_raw': _bytes_feature(convert_to_raw(ezr_org)),
            'ezi_raw': _bytes_feature(convert_to_raw(ezi_org))
            }))
        tfwriter.write(example.SerializeToString())


res = glob.glob("../run*/IntField-Y_170709.tfrecords")
with tf.python_io.TFRecordWriter(OUT_FILE) as tf_writer:
    for idx, elem in enumerate(res):
        #  for debug
        # if idx >= 2:
        #     sys.exit()

        print("idx%d:%s" % (idx, elem))
        combineTFRecords(elem, tf_writer)

テスト読込みcode

まとめたファイルがきちんと読めるか確認するためのコード。

test_readCombined_170722.py

import numpy as np
import tensorflow as tf

"""
v0.2 Jul. 09, 2017
  - read [mr] and [mi]
v0.1 Jul. 09, 2017
  - read position and Ex, Ey, Ez
     + add get_feature_float32()
"""

# on
#   Ubuntu 16.04 LTS
#   TensorFlow v1.1
#   Python 3.5.2

# codingrule: PEP8


def get_feature_float32(example, feature_name):
    wrk_raw = (example.features.feature[feature_name]
               .bytes_list
               .value[0])
    wrk_1d = np.fromstring(wrk_raw, dtype=np.float32)
    wrk_org = wrk_1d.reshape([1, -1])
    return wrk_org

INP_FILE = 'combined_IntField-Y_170722.tfrecords'

record_iterator = tf.python_io.tf_record_iterator(path=INP_FILE)
for record in record_iterator:
    example = tf.train.Example()
    example.ParseFromString(record)

    xpos_org = get_feature_float32(example, 'xpos_raw')
    ypos_org = get_feature_float32(example, 'ypos_raw')
    zpos_org = get_feature_float32(example, 'zpos_raw')
    mr_org = get_feature_float32(example, 'mr_raw')
    mi_org = get_feature_float32(example, 'mi_raw')
    exr_org = get_feature_float32(example, 'exr_raw')
    exi_org = get_feature_float32(example, 'exi_raw')
    eyr_org = get_feature_float32(example, 'eyr_raw')
    eyi_org = get_feature_float32(example, 'eyi_raw')
    ezr_org = get_feature_float32(example, 'ezr_raw')
    ezi_org = get_feature_float32(example, 'ezi_raw')

    list_pos = *xpos_org, *ypos_org, *zpos_org, *mr_org, *mi_org
    list_e = *exr_org, *exi_org, *eyr_org, *eyi_org, *ezr_org, *ezi_org

    print(*list_pos, *list_e)
    #print(*ezi_org)

実行

$ python3 combine_TFRecords_170722.py 
idx0:../run366_sphere_g26_m1.45/IntField-Y_170709.tfrecords
idx1:../run375_sphere_g26_m1.5/IntField-Y_170709.tfrecords
idx2:../run354_sphere_g26_m1.33/IntField-Y_170709.tfrecords
idx3:../run362_sphere_g26_m1.4/IntField-Y_170709.tfrecords
idx4:../run364_sphere_g26_m1.4/IntField-Y_170709.tfrecords
idx5:../run368_sphere_g26_m1.45/IntField-Y_170709.tfrecords
idx6:../run370_sphere_g26_m1.45/IntField-Y_170709.tfrecords
idx7:../run376_sphere_g26_m1.5/IntField-Y_170709.tfrecords
idx8:../run367_sphere_g26_m1.45/IntField-Y_170709.tfrecords
idx9:../run356_sphere_g26_m1.33/IntField-Y_170709.tfrecords
idx10:../run372_sphere_g26_m1.5/IntField-Y_170709.tfrecords
idx11:../run355_sphere_g26_m1.33/IntField-Y_170709.tfrecords
idx12:../run360_sphere_g26_m1.4/IntField-Y_170709.tfrecords
idx13:../run373_sphere_g26_m1.5/IntField-Y_170709.tfrecords
idx14:../run365_sphere_g26_m1.45/IntField-Y_170709.tfrecords
idx15:../run369_sphere_g26_m1.45/IntField-Y_170709.tfrecords
idx16:../run353_sphere_g26_m1.33/IntField-Y_170709.tfrecords
idx17:../run357_sphere_g26_m1.33/IntField-Y_170709.tfrecords
idx18:../run371_sphere_g26_m1.5/IntField-Y_170709.tfrecords
idx19:../run359_sphere_g26_m1.4/IntField-Y_170709.tfrecords
idx20:../run358_sphere_g26_m1.33/IntField-Y_170709.tfrecords
idx21:../run374_sphere_g26_m1.5/IntField-Y_170709.tfrecords
idx22:../run363_sphere_g26_m1.4/IntField-Y_170709.tfrecords
idx23:../run361_sphere_g26_m1.4/IntField-Y_170709.tfrecords

$ ls -l combined_IntField-Y_170722.tfrecords 
-rw-rw-r-- 1 xxx xxx 56191872  7月 22 09:31 combined_IntField-Y_170722.tfrecords

$ python3 test_readCombined_170722.py | head -n 5
[-0.21666151] [-1.51663053] [-5.41653776] [ 1.45000005] [ 0.001] [-0.02894282] [ 0.02603715] [ 0.17363222] [ 0.70102149] [-0.09119416] [-0.12842615]
[ 0.21666151] [-1.51663053] [-5.41653776] [ 1.45000005] [ 0.001] [ 0.02894282] [-0.02603715] [ 0.17363222] [ 0.70102149] [-0.09119417] [-0.12842615]
[-1.0833075] [-1.0833075] [-5.41653776] [ 1.45000005] [ 0.001] [-0.11953837] [ 0.11888158] [ 0.32258603] [ 0.59490114] [-0.06808119] [-0.09587516]
[-0.64998454] [-1.0833075] [-5.41653776] [ 1.45000005] [ 0.001] [-0.11078756] [ 0.05728684] [ 0.17507839] [ 0.67870367] [-0.06764975] [-0.1068899]
[-0.21666151] [-1.0833075] [-5.41653776] [ 1.45000005] [ 0.001] [-0.04282469] [ 0.00599804] [ 0.12292478] [ 0.8378399] [-0.0632441] [-0.10424931]
Traceback (most recent call last):
  File "test_readCombined_170722.py", line 50, in <module>
    print(*list_pos, *list_e)
BrokenPipeError: [Errno 32] Broken pipe

$ python3 test_readCombined_170722.py | tail -n 5
[ 0.22439376] [ 1.12196875] [ 5.60984373] [ 1.39999998] [ 0.01] [-0.03255619] [ 0.01280934] [-3.08298993] [-0.54999858] [-0.09020738] [ 1.30495083]
[ 0.67318124] [ 1.12196875] [ 5.60984373] [ 1.39999998] [ 0.01] [-0.22019333] [ 0.02005777] [-2.49291277] [-0.39416489] [-0.08645402] [ 1.29670262]
[ 1.12196875] [ 1.12196875] [ 5.60984373] [ 1.39999998] [ 0.01] [-0.42727378] [ 0.01380414] [-2.04840279] [-0.19202116] [-0.09770373] [ 1.21033025]
[-0.22439376] [ 1.57075632] [ 5.60984373] [ 1.39999998] [ 0.01] [ 0.09677245] [-0.01133237] [-2.4711504] [-0.21153302] [-0.13845336] [ 1.68478251]
[ 0.22439376] [ 1.57075632] [ 5.60984373] [ 1.39999998] [ 0.01] [-0.09677245] [ 0.01133237] [-2.4711504] [-0.21153302] [-0.13845336] [ 1.68478251]

refractive indexの値を見ると、まとめることができているようだ。

サンプル数

$ python3 test_readCombined_170722.py | wc
 223872 3916020 32910582