Environment

GeForce GTX 1070 (8GB)
ASRock Z170M Pro4S [Intel Z170chipset]
Ubuntu 14.04 LTS desktop amd64
TensorFlow v0.11
cuDNN v5.1 for Linux
CUDA v8.0
Python 2.7.6
IPython 5.1.0 -- An enhanced Interactive Python.
gcc (Ubuntu 4.8.4-2ubuntu1~14.04.3) 4.8.4
GNU bash, version 4.3.8(1)-release (x86_64-pc-linux-gnu)

This article is related to ADDA (light scattering simulator based on the discrete dipole approximation).

previous: v0.2

Reading files

chp.0 : checkpoint file
- obtained by original [iterative.c]
aux.0 : auxiliary info
- obtained by modified [iterative.c]
- iterative.c > v0.4 > modification to save an auxiliary file and a coordinate file

code v0.5

Refactoring by adding functions and using named tuples for the return of the functions.

ref (in Japanese): http://qiita.com/7of9/items/6d37e362f7be275d2f9a

UtilReadCheckPoint.py

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import array
import sys
import collections as clc

'''
v0.5 Apr. 09, 2017
    - read_chpoint_file() is called with [debugPrint=False]
    - add debug_group_print()
    - read_chpoint_file() returns in groups
        + IterativeGroup
        + AuxGroup
        + VectorGroup
v0.4 Apr. 09, 2017
    - read_auxiliary_info() returns named tuple
    - read_vectorTuple() returns named tuple
v0.3 Apr. 07, 2017
    - read_iterative_info() returns named tuple
    - import collections
    - add read_iterative_info()
    - add read_vectorTuple()
v0.2 Apr. 06, 2017
    - add debug_print_vectors()
    - add debug_print_scalarsList()
    - add debug_print_sizes()
    - add debug_print_scalars1()
v0.1 Apr. 05, 2017
    - branched
===== branched from [read_chpoint.py] to [UtilReadCheckPoint.py] =====
v0.6 Dec. 23, 2016
    - try to print fractional part of xvec[] in 4 digits
       + xvec[] = np.array([drel, dimg]), also for rvec[], pvec[], vectors[][]
       + add [np.set_printoptions(precision=4)]
v0.5 Dec. 23, 2016
    - read scalars[], xvec[], rvec[], pvec[], vectors[][]
    - refactor for PEP8 (except for E501 line too long for long comment)
    - wrap_fromfile() takes [num] arg
v0.4 Dec. 21, 2016
    - read from "auxiliary" file [auxfp]
        + add read_auxiliary_info()
v0.3 Dec. 19, 2016
    - add wrap_fromfile()
    - read [inprodR],[prev_err],[resid_scale]
v0.2 Dec. 19, 2016
    - fix bug > read [local_nRows] for "size_t" type
v0.1 Dec. 18, 2016
    - read_chpoint_file() > read [ind_m],[local_nRows],[niter],[counter]
'''

# codingrule:PEP8
# on Python 2.7.6

np.set_printoptions(precision=6)


def wrap_fromfile(rfp, typecode, num):
    res = array.array(typecode)
    res.fromfile(rfp, num)
    return res


def read_auxiliary_info(auxFilename):
    with open(auxFilename, "rb") as auxfp:
        sc_N = wrap_fromfile(auxfp, 'i', num=1)  # the number of scalars' sizes

        sc_sizes = array.array('i')  #
        sc_sizes.fromfile(auxfp, *sc_N)  # '*':unpack list

        vec_N = wrap_fromfile(auxfp, 'i', num=1)  # number of vercotrs' sizes

        vec_sizes = array.array('i')  #
        vec_sizes.fromfile(auxfp, *vec_N)  # '*':unpack list

        AuxGroup = clc.namedtuple('AuxGroup',
                                  ['sc_N', 'sc_sizes', 'vec_N', 'vec_sizes'])
        auxgrp = AuxGroup(sc_N, sc_sizes, vec_N, vec_sizes)
        return auxgrp
    # TODO: 0m > error handling when file is NULL


def debug_print_scalars1(ind_m, local_nRows, niter, counter,
                         inprodR, prev_err, resid_scale):
    print('ind_m:', ind_m)
    print('local_nRows:', local_nRows)
    print('niter:', niter)
    print('counter:', counter)
    print('inprodR:', inprodR)
    print('prev_err:', prev_err)
    print('resid_scale:', resid_scale)  # scale to get square of relative error


def debug_print_sizes(sc_N, sc_sizes, vec_N, vec_sizes):
    print('sc_N:', sc_N)
    print('sc_sizes', sc_sizes)
    print('vec_N:', vec_N)
    print('vec_sizes:', vec_sizes)


def debug_print_scalarsList(sc_N_zero, scalars):
    for idx in range(sc_N_zero):
        print('scalars:', scalars[idx])


def debug_print_vectors(xvec, rvec, pvec, vectors):
    for idx in range(3):
        print('xvec:', np.array_repr(xvec[idx]).replace('\n', ''))
        print('rvec:', np.array_repr(rvec[idx]).replace('\n', ''))
        print('pvec:', np.array_repr(pvec[idx]).replace('\n', ''))
        print('vectors:', np.array_repr(vectors[idx][0]).replace('\n', ''))
        print('vectors:', np.array_repr(vectors[idx][1]).replace('\n', ''))
        print('vectors:', np.array_repr(vectors[idx][2]).replace('\n', ''))


def debug_group_print(itrgrp, auxgrp, vecgrp):
    debug_print_scalars1(itrgrp.ind_m, itrgrp.local_nRows,
                         itrgrp.niter, itrgrp.counter,
                         itrgrp.inprodR, itrgrp.prev_err,
                         itrgrp.resid_scale)
    debug_print_sizes(auxgrp.sc_N, auxgrp.sc_sizes, auxgrp.vec_N,
                      auxgrp.vec_sizes)
    debug_print_scalarsList(auxgrp.sc_N[0], vecgrp.scalars)
    debug_print_vectors(vecgrp.xvec, vecgrp.rvec, vecgrp.pvec,
                        vecgrp.vectors)


def read_vectorTuple(chpfp, sc_N, sc_sizes, vec_N, vec_sizes, local_nRows):
        # scalars[]
        scalars = []
        for idx in range(sc_N[0]):
            if sc_sizes[idx] == 8:  # double
                scl = wrap_fromfile(chpfp, 'd', num=1)
                scalars.append(scl)
            elif sc_sizes[idx] == 16:  # double complex
                scl = wrap_fromfile(chpfp, 'd', num=2)
                scalars.append(scl)

        # xvec[] : total electric field on the dipoles
        # TODO: 0m > replace 'local_nRows[0]' with function?
        # TODO: 0m > replace 'vec_N[0]' with function?
        # TODO: 0m > define function for read [drel,dimg]??
        xvec = [[] for idx in range(local_nRows[0])]
        for idx in range(local_nRows[0]):
            drel = wrap_fromfile(chpfp, 'd', 1)
            dimg = wrap_fromfile(chpfp, 'd', 1)
            xvec[idx] = np.array([drel, dimg])

        # rvec[] : current residual
        rvec = [[] for idx in range(local_nRows[0])]
        for idx in range(local_nRows[0]):
            drel = wrap_fromfile(chpfp, 'd', 1)
            dimg = wrap_fromfile(chpfp, 'd', 1)
            rvec[idx] = np.array([drel, dimg])

        # pvec[] : polarization of dipoles, also an auxiliary vector
        #   in iterative solvers
        pvec = [[] for idx in range(local_nRows[0])]
        for idx in range(local_nRows[0]):
            drel = wrap_fromfile(chpfp, 'd', 1)
            dimg = wrap_fromfile(chpfp, 'd', 1)
            pvec[idx] = np.array([drel, dimg])

        # vectors[] : ???
        MAXNUM_VECTORS = 20
        vectors = [[] for idx in range(MAXNUM_VECTORS)]
        for idx_vecN in range(vec_N[0]):
            alist = []
            for idx_LnRows in range(local_nRows[0]):
                if vec_sizes[idx_vecN] == 16:
                    drel = wrap_fromfile(chpfp, 'd', 1)
                    dimg = wrap_fromfile(chpfp, 'd', 1)
                    alist.append(np.array([drel, dimg]))
            vectors[idx_vecN] = alist
        VectorGroup = clc.namedtuple('VectorGroup',
                                     ['scalars', 'xvec', 'rvec',
                                      'pvec', 'vectors'])
        vecgrp = VectorGroup(scalars, xvec, rvec, pvec, vectors)
        return vecgrp


def read_iterative_info(chpfp):
    ind_m = wrap_fromfile(chpfp, 'i', num=1)  # index of iterative method
    # number of local rows of decomposition (only real dipoles)
    #    where L: unsigned long = size_t
    local_nRows = wrap_fromfile(chpfp, 'L', num=1)
    niter = wrap_fromfile(chpfp, 'i', num=1)  # iteration count
    # number of successive iterations without residual decrease
    counter = wrap_fromfile(chpfp, 'i', num=1)
    # used as |r_0|^2 and best squared norm of residual up to
    #   some iteration
    inprodR = wrap_fromfile(chpfp, 'd', num=1)
    # previous relative error; used in ProgressReport, initialized
    #   in IterativeSolver
    prev_err = wrap_fromfile(chpfp, 'd', num=1)
    # scale to get square of relative error
    resid_scale = wrap_fromfile(chpfp, 'd', num=1)
    #
    IterGroup = clc.namedtuple('IterativeGroup',
                               ['ind_m', 'local_nRows', 'niter', 'counter',
                                'inprodR', 'prev_err', 'resid_scale'])
    itrgrp = IterGroup(ind_m, local_nRows, niter, counter,
                       inprodR, prev_err, resid_scale)
    return itrgrp


def read_chpoint_file(chpFilename, auxFilename, debugPrint=False):
    if debugPrint:
        print(chpFilename)
        print(auxFilename)

    # TODO: 0m > error handling when file is NULL
    with open(chpFilename, "rb") as chpfp:
        # 1. iterative information
        itrgrp = read_iterative_info(chpfp)
        if debugPrint:
            debug_print_scalars1(itrgrp.ind_m, itrgrp.local_nRows,
                                 itrgrp.niter, itrgrp.counter,
                                 itrgrp.inprodR, itrgrp.prev_err,
                                 itrgrp.resid_scale)

        # 2. auxiliary information
        #
        # sc_N: the number of scalars' sizes
        # sc_sizes:
        # vec_N: the number of vercotrs' sizes
        # vec_sizes:
        auxgrp = read_auxiliary_info(auxFilename)
        if debugPrint:
            debug_print_sizes(auxgrp.sc_N, auxgrp.sc_sizes, auxgrp.vec_N,
                              auxgrp.vec_sizes)

        # 3. vector list
        vecgrp = read_vectorTuple(chpfp, auxgrp.sc_N, auxgrp.sc_sizes,
                                  auxgrp.vec_N, auxgrp.vec_sizes,
                                  itrgrp.local_nRows)

        #
        if debugPrint:
            debug_print_scalarsList(auxgrp.sc_N[0], vecgrp.scalars)
        if debugPrint:
            debug_print_vectors(vecgrp.xvec, vecgrp.rvec, vecgrp.pvec,
                                vecgrp.vectors)

        return itrgrp, auxgrp, vecgrp


if __name__ == '__main__':
    argvs = sys.argv
    argc = len(argvs)

    # print(argvs)
    if (argc < 3):
        print("ERROR: chpoint file is not specified\r\n")
        print("   [cmd] [chpoint file] [auxiliary file]\r\n")
        sys.exit()

    res = read_chpoint_file(chpFilename=argvs[1], auxFilename=argvs[2],
                            debugPrint=False)
    itrgrp, auxgrp, vecgrp = res

    # debug
    debug_group_print(itrgrp, auxgrp, vecgrp)

run

$ python UtilReadCheckPoint.py chp.0 aux.0 #run
ind_m: array('i', [5])
local_nRows: array('L', [27984L])
niter: array('i', [210])
counter: array('i', [140])
inprodR: array('d', [4.317011265934948])
prev_err: array('d', [0.31977234550768563])
resid_scale: array('d', [0.020384616751203153])
sc_N: array('i', [8])
sc_sizes array('i', [8, 8, 8, 8, 16, 16, 16, 16])
vec_N: array('i', [3])
vec_sizes: array('i', [16, 16, 16])
scalars: array('d', [21.695179646708983])
scalars: array('d', [17.561818003907533])
scalars: array('d', [0.01622964090344006])
scalars: array('d', [0.01658574452030678])
scalars: array('d', [2.6036437445821585, 3.735208975819251])
scalars: array('d', [-0.16158984774519294, 0.33676364827430344])
scalars: array('d', [0.34430517188699605, -0.9387175013645006])
scalars: array('d', [-0.39247820060720345, -0.9196117523862111])
xvec: array([[ 0.000935],       [-0.001522]])
rvec: array([[-0.002113],       [ 0.002774]])
pvec: array([[ 0.007899],       [ 0.002861]])
vectors: array([[-0.203491],       [ 0.033598]])
vectors: array([[ 0.513358],       [-0.235685]])
vectors: array([[ 0.548095],       [ 0.061967]])
xvec: array([[ 0.03277],       [ 0.09079]])
rvec: array([[ 0.004444],       [-0.022859]])
pvec: array([[-0.020303],       [-0.017681]])
vectors: array([[-0.114939],       [ 0.099301]])
vectors: array([[ 0.310234],       [-0.340468]])
vectors: array([[ 0.330688],       [-0.145048]])
xvec: array([[-0.004644],       [-0.000675]])
rvec: array([[ 0.005379],       [-0.006603]])
pvec: array([[-0.019206],       [-0.010089]])
vectors: array([[-0.000433],       [ 0.008886]])
vectors: array([[ 0.007084],       [-0.024328]])
vectors: array([[ 0.006453],       [-0.022867]])