30秒に１度自動更新が掛かるあるサイトに対してのスクレイピングが最終ページまで動作しない

const livePage = `https://hogehoge.co.jp/hoge/${pageId}/page?index=0110100`
var indexCount = 0;
var redirectCount = 0;
const page = await browser.newPage();

var dataDir = __dirname + '/data';

await page.goto(livePage);
await page.waitFor(1000);

var oldNext;
var next;

var lastHtml = '';
var loop = true;
var beforeStartCount = 0;
var loopCount = 0;
do {
	var manual = await page.$x('//*[@id="btn_manual"]/a');
	if (manual && manual[0]) {
		try {
			await manual[0].click();
		} catch(err) {
		}
	}

	var liveBody = await page.$x('//body');
	var liveHtml = await page.evaluate(e => e.innerHTML, liveBody[0]);
	if (liveHtml != lastHtml) {
		indexCount++;
		fs.writeFileSync(`${dataDir}/${pageId}.live.${indexCount}.html`, liveHtml);
	}
	
	oldNext = next;
	next = await page.$x('//a[@id="btn_next" and @href]');
	if (next && next.length > 0) {
		redirectCount = 0;
		try {
			await next[0].click();
		} catch(err) {
		}
		await page.waitFor(1000);
		continue;
	}
	else
	{
		redirectCount++;
		if( redirectCount < 5 )
		{
			if (oldNext && oldNext.length > 0) {
				indexCount--;
				next = oldNext;
				await page.waitFor(10000);
				try {
					await oldNext[0].click();
				} catch(err) {
				}
				await page.waitFor(1000);
				continue;
			}
		}
	}
	await page.waitFor(1000 + Math.random() * 2000);

	var refresh = await page.$x('//a[@id="btn_reflesh"]');
	try {
		await refresh[0].click();
	} catch(err) {
	}

	loop = !(await isGameSet(page));
	loopCount++;
} while(loop && loopCount < 600);