LoginSignup
1
0

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?

More than 5 years have passed since last update.

@harhid(Hidehiro Haranaga)

大量商用ページの表示結果を比較するために試行錯誤した(２)

Last updated at Posted at 2019-09-09

元記事はこちら

大人の事情で公開していなかった部分を含めて、全てみせます。
自分の記録の意味も含めて。

screenshotDiff.js
/*------------------------------------------------------------
   大量ページを逐次表示、スクリーンショットをとり
   ペアページ同士で比較差分を表示する。

   制限事項： 自動アクセス防止ヒューリスティックを採用して
   いるサイトは アクセス権限無し(403)を返すため比較差分を
   することができない。

	node v8.4.0
	puppeteer-core v1.8.0
	looks-same v4.0.0
	@I.Times  2018/9/25 
--------------------------------------------------------------*/
/*------------------------------------------------------------
   2018/9/24 I.Times Haranaga
   puppeteer によるスクリーンショットの工夫 
   (1) ページ表示後にスクリーンショットを取らず一度リロードを
       してからスクリーンショットをとる。
       リロード後のほうがページ描画が安定するため。

--------------------------------------------------------------*/
/*------------------------------------------------------------
   2018/9/24 I.Times Haranaga
   looks-sameには独自修正を加えている。
   (1) createDiff実行時に 差分ピクセル数を返すようCallBackを修正
　 (2) 差分箇所のみに点を描画する画像ファイルを追加作成する。

--------------------------------------------------------------*/
const puppeteer = require('puppeteer-core');
const async = require('async');
const delay = require('delay');
const fs = require('fs');
const che = require('cheerio');
var looksSame = require('looks-same');

var os = require('os');

// 数値を前ゼロ埋めする。
function zeroPadding(num,length){
	var ZERO = '0000000000000000';
	if(length){
		if(length > ZERO.length){
			return (ZERO + num).slice(ZERO.length*(-1));
		}else if(length > 0){
			return (ZERO + num).slice(-length);
		}else{
			return num;
		}
	}else{
		return (ZERO + num).slice(ZERO.length*(-1));
	}

}

const headless = true;

const browserExecutablePath = 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe';
const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36';
//const userAgent = 'BOT/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36';
const WindowWidth = 1920;
const WindowHeight = 1080;

const viewPort = {
	width: WindowWidth, 
	height: WindowHeight, 
	deviceScaleFactor:0.85, 
	isMobile:false, 
	Mobile:false, 
	hasTouch:false, 
	isLandscape:false,
};

const NO_SANDBOX = '--no-sandbox';

const _browserOptions = {
	headless: headless,
	ignoreHTTPSErrors: true,
	executablePath: browserExecutablePath,
	defaultViewport:viewPort,
	args: ['--window-size='+WindowWidth+','+WindowHeight,'--window-position=0,0',NO_SANDBOX],
};

var browseOptions = _browserOptions;

// Async-Awaitの中の例外をキャッチする仕組み
process.on('unhandledRejection', console.dir);

const DIRNAME = __dirname;


(async () => {

	// URL一覧を読み込む。
	var elements = [];
	var elementList = async () => {
		var xml_data = await fs.readFileSync("data.xml", "utf-8");
		$ = che.load(xml_data);
		$("element").each(function(i0, el0) {
			var elem = {};
			var actualElem = {};
			var actual = $(this).children("actual");
			actualElem['url'] = actual.children("url").text();
			actualElem['networkidle'] = $(actual).children("networkidle").text();
			actualElem['delay'] = $(actual).children("delay").text();
			elem['actual'] = actualElem;
			var targetElem = {};
			var target = $(this).children("target");
			targetElem['url'] = target.children("url").text()
			targetElem['networkidle'] = target.children("networkidle").text()
			targetElem['delay'] = target.children("delay").text()
			elem['target'] = targetElem;
			elem['tolerance'] = $(this).children('tolerance').text();
			elements.push(elem);
		});
	};
	elementList();

	// スクリーンショットFunction定義：ページを表示,スクリーンショットを取る。
	var screenShot = async  (element, shotPath) =>{
		await page.setUserAgent(userAgent);
		await page.goto(element.url,  {waitUntil: element.networkidle})
		.then( async function(response){
			// ReLoadする理由
			// ReLoad再描画が時間が掛からず、ScreenShotタイミングを取りやすいため
			
			await page.reload({waitUntil: element.networkidle}).then( async function(response) {
				if(element.delay>0){
					await delay(element.delay);
				}
				await page.screenshot( {path: DIRNAME+'\\'+shotPath, fullPage: true});
			});
		} );
	};

	// 配列要素を処理するAsync Function定義
	var eachProcess = async (element, callback) => {

		var index = elements.indexOf( element );
		var count = index + 1;
		var path = {
			imageA: 'imageA\\shotA_'+zeroPadding(count,5)+'.png',
			imageB: 'imageB\\shotB_'+zeroPadding(count,5)+'.png'
		};
		// Actual スクリーンショット
		await screenShot(element.actual, path.imageA);
		// Target スクリーンショット
		await screenShot(element.target, path.imageB);
		// Actual<=>Targetの比較
		var looksSameOption = {};
		looksSameOption.reference = path.imageA;
		looksSameOption.current = path.imageB;
		looksSameOption.diff = 'diff\\diff_'+zeroPadding(count,5)+'_1.png';
		looksSameOption.diff2 = 'diff\\diff_'+zeroPadding(count,5)+'_2.png'; // 独自追加オプション
		looksSameOption.highlightColor = '#ff00ff'; //color to highlight the differences
		looksSameOption.defaultColor = '#ffffff'; // 独自追加オプション
		looksSameOption.strict = true; //strict comparsion
		looksSameOption.writeOriginalDiff = true; // 独自追加オプション

		looksSame.createDiff(looksSameOption
		// このパラメータFunctionは独自追加です。
		,function(unmatch){
			console.log("No.["+zeroPadding(count,5)+"] UnMatch["+zeroPadding(unmatch)+"]:"+element.actual.url);
			// 必ずここのfunctionを一度呼びだすので、ここで forEachSeries のcallbackを呼び出す。
			callback();
		// このパラメータFunctionは独自追加です。
		}, function(err){
			if(err){
				console.log(err);
				throw err;
			}
		}, function(err){
			if(err){
				console.log(err);
				throw err;
			}
		} ); // looksSame 終わり
	};

	// ブラウザを起動する。
	var browser = await puppeteer.launch( _browserOptions );
	const page = await browser.newPage();
	await page.setJavaScriptEnabled(true);
	//await page.evaluate('navigator.userAgent');
	// キャッシュ無効にする（効果は未確認）
	const client = await page.target().createCDPSession();
	await client.send( 'Network.setCacheDisabled', { 'cacheDisabled' : true } );
	await page.setCacheEnabled( false );

	// 配列(elements)の要素をAsync順次処理する。
	async.forEachSeries( 
		// 第一パラメータ：配列
		elements, 
		// 第二パラメータ：要素を処理するAsync Function
		(async function(element, callback){
		
			await eachProcess(element, callback);
		}),
		// 第三パラメータ： 最後に呼び出されるCallBack
		async function(err){
			if(err) throw err;

			await browser.close();
			console.log('##browser close');
		}
	); // async.forEachSeries終わり
})();
data.xml(例)
<element>
	<actual>
		<url>https://www.naro.affrc.go.jp/nivfs/index.html</url>
		<networkidle>networkidle0</networkidle>
		<delay>0</delay>
	</actual>
	<target>
		<url>https://www.naro.affrc.go.jp/nivfs/index.html</url>
		<networkidle>networkidle0</networkidle>
		<delay>0</delay>
	</target>
	<tolerance>0</tolerance>
</element>
<element>
	<actual>
		<url>https://www.paxcompy.co.jp/</url>
		<networkidle>networkidle0</networkidle>
		<delay>0</delay>
	</actual>
	<target>
		<url>https://www.paxcompy.co.jp/</url>
		<networkidle>networkidle0</networkidle>
		<delay>0</delay>
	</target>
	<tolerance>0</tolerance>
</element>
<element>
	<actual>
		<url>http://www.machidukuri-nagano.jp/</url>
		<networkidle>networkidle0</networkidle>
		<delay>0</delay>
	</actual>
	<target>
		<url>http://www.machidukuri-nagano.jp/</url>
		<networkidle>networkidle0</networkidle>
		<delay>0</delay>
	</target>
	<tolerance>0</tolerance>
</element>
node_modules\looks-same\index.js
'use strict';

const _ = require('lodash');
const parseColor = require('parse-color');
const colorDiff = require('color-diff');
const png = require('./lib/png');
const areColorsSame = require('./lib/same-colors');
const AntialiasingComparator = require('./lib/antialiasing-comparator');
const IgnoreCaretComparator = require('./lib/ignore-caret-comparator');
const utils = require('./lib/utils');
const readPair = utils.readPair;
const getDiffPixelsCoords = utils.getDiffPixelsCoords;

const JND = 2.3; // Just noticeable difference if ciede2000 >= JND then colors difference is noticeable by human eye

const getDiffArea = (diffPixelsCoords) => {
    const xs = [];
    const ys = [];

    diffPixelsCoords.forEach((coords) => {
        xs.push(coords[0]);
        ys.push(coords[1]);
    });

    const top = Math.min.apply(Math, ys);
    const bottom = Math.max.apply(Math, ys);

    const left = Math.min.apply(Math, xs);
    const right = Math.max.apply(Math, xs);

    const width = (right - left) + 1;
    const height = (bottom - top) + 1;

    return {left, top, width, height};
};

const makeAntialiasingComparator = (comparator, png1, png2, opts) => {
    const antialiasingComparator = new AntialiasingComparator(comparator, png1, png2, opts);
    return (data) => antialiasingComparator.compare(data);
};

const makeNoCaretColorComparator = (comparator, pixelRatio) => {
    const caretComparator = new IgnoreCaretComparator(comparator, pixelRatio);
    return (data) => caretComparator.compare(data);
};

function makeCIEDE2000Comparator(tolerance) {
    return function doColorsLookSame(data) {
        if (areColorsSame(data)) {
            return true;
        }
        /*jshint camelcase:false*/
        const lab1 = colorDiff.rgb_to_lab(data.color1);
        const lab2 = colorDiff.rgb_to_lab(data.color2);

        return colorDiff.diff(lab1, lab2) < tolerance;
    };
}

const createComparator = (png1, png2, opts) => {
    let comparator = opts.strict ? areColorsSame : makeCIEDE2000Comparator(opts.tolerance);

    if (opts.ignoreAntialiasing) {
        comparator = makeAntialiasingComparator(comparator, png1, png2, opts);
    }

    if (opts.ignoreCaret) {
        comparator = makeNoCaretColorComparator(comparator, opts.pixelRatio);
    }

    return comparator;
};

const iterateRect = (width, height, callback, endCallback) => {
    const processRow = (y) => {
        setImmediate(() => {
            for (let x = 0; x < width; x++) {
                callback(x, y);
            }

            y++;

            if (y < height) {
                processRow(y);
            } else {
                endCallback();
            }
        });
    };

    processRow(0);
};

const buildDiffImage = (png1, png2, options, callback) => {
    const width = Math.max(png1.width, png2.width);
    const height = Math.max(png1.height, png2.height);
    const minWidth = Math.min(png1.width, png2.width);
    const minHeight = Math.min(png1.height, png2.height);
    const highlightColor = options.highlightColor;
    const result = png.empty(width, height);

    // ###### ここを変えた ########
    // -- add start ---
    const result2 = (options.writeOriginalDiff)? png.empty(width, height): null;
    var unmatch = 0;
    // -- add end ---

    iterateRect(width, height, (x, y) => {
        if (x >= minWidth || y >= minHeight) {
            result.setPixel(x, y, highlightColor);
	    // ###### ここを変えた ########
	    // -- add start ---
	    unmatch += 1;
	    // -- add end ---
            return;
        }

        const color1 = png1.getPixel(x, y);
        const color2 = png2.getPixel(x, y);

        if (!options.comparator({color1, color2})) {
            result.setPixel(x, y, highlightColor);
            // ###### ここを変えた ########
            // -- add start ---
            unmatch += 1;
	    if(options.writeOriginalDiff){
		result2.setPixel(x, y, options.highlightColor);
	    }
            // -- add end ---
        } else {
	    result.setPixel(x, y, color1);
	    // ###### ここを変えた ########
	    // add start
	    if(options.writeOriginalDiff){
		result2.setPixel(x, y, options.defaultColor);
	    }
	    // add end
        }
    // ###### ここを変えた ########
    //}, () => callback(result));
    }, () => callback(result, result2, unmatch));
};

const parseColorString = (str) => {
    const parsed = parseColor(str);

    return {
        R: parsed.rgb[0],
        G: parsed.rgb[1],
        B: parsed.rgb[2]
    };
};

const getToleranceFromOpts = (opts) => {
    if (!_.hasIn(opts, 'tolerance')) {
        return JND;
    }

    if (opts.strict) {
        throw new TypeError('Unable to use "strict" and "tolerance" options together');
    }

    return opts.tolerance;
};

const prepareOpts = (opts) => {
    opts.tolerance = getToleranceFromOpts(opts);

    _.defaults(opts, {
        ignoreAntialiasing: true,
        antialiasingTolerance: 0
    });
};

module.exports = exports = function looksSame(reference, image, opts, callback) {
    if (!callback) {
        callback = opts;
        opts = {};
    }

    prepareOpts(opts);

    readPair(reference, image, (error, pair) => {
        if (error) {
            return callback(error);
        }

        const first = pair.first;
        const second = pair.second;

        if (first.width !== second.width || first.height !== second.height) {
            return process.nextTick(() => callback(null, false));
        }

        const comparator = createComparator(first, second, opts);

        getDiffPixelsCoords(first, second, comparator, {stopOnFirstFail: true}, (result) => {
            callback(null, result.length === 0);
        });
    });
};

exports.getDiffArea = function(reference, image, opts, callback) {
    if (!callback) {
        callback = opts;
        opts = {};
    }

    prepareOpts(opts);

    readPair(reference, image, (error, pair) => {
        if (error) {
            return callback(error);
        }

        const first = pair.first;
        const second = pair.second;

        if (first.width !== second.width || first.height !== second.height) {
            return process.nextTick(() => callback(null, {
                width: Math.max(first.width, second.width),
                height: Math.max(first.height, second.height),
                top: 0,
                left: 0
            }));
        }

        const comparator = createComparator(first, second, opts);

        getDiffPixelsCoords(first, second, comparator, (result) => {
            if (!result.length) {
                return callback(null, null);
            }

            callback(null, getDiffArea(result));
        });
    });
};

//### ここ変えた！
//exports.createDiff = function saveDiff(opts, callback) {
exports.createDiff = function saveDiff(opts, callback, callback2, callback3) {
    const tolerance = getToleranceFromOpts(opts);

    readPair(opts.reference, opts.current, (error, result) => {
        if (error) {
            return callback(error);
        }

        const diffOptions = {
            highlightColor: parseColorString(opts.highlightColor),
	    // ### 下記１行追加
	    defaultColor : parseColorString(opts.defaultColor),
	    writeOriginalDiff: opts.writeOriginalDiff,
            comparator: opts.strict ? areColorsSame : makeCIEDE2000Comparator(tolerance)
        };

        // ### ここ変えた！！	
	//buildDiffImage(result.first, result.second, diffOptions, (result) => {
        buildDiffImage(result.first, result.second, diffOptions, (result, result2, unmatch) => {
            // ### ここ変えた！！	
            //if (opts.diff === undefined) {
            //    result.createBuffer(callback);
            //} else {
            //    result.save(opts.diff, callback);
	    //}
            if (opts.diff === undefined) {
                result.createBuffer(callback2);
            } else {
                result.save(opts.diff, callback2);
		if(opts.diff2 && result2){
	                result2.save(opts.diff2, callback3);
		}
		callback(unmatch);
            }
        });
    });
};

exports.colors = (color1, color2, opts) => {
    opts = opts || {};

    if (opts.tolerance === undefined) {
        opts.tolerance = JND;
    }

    const comparator = makeCIEDE2000Comparator(opts.tolerance);

    return comparator({color1, color2});
};
1
0
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
1
0

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?