Wikisource から検証済の画像と文字起こしデータを取得する

Wikisourceの文字起こし前の画像の取得が少し難しかったので書いておきます。

import mwclient
import requests

user_agent = 'XXX/0.1'
site = mwclient.Site("ja.wikisource.org", clients_useragent=user_agent) 

def get_thumb_url(page: mwclient.page.Page) -> str:
    page_num = int(page.name.split('/')[-1])
    for img in page.images(True):
        r = site.api("query", prop="imageinfo", titles=img.name, iiurlparam = f"page{page_num}-1px", iiprop="url", iiurlwidth="1280", formatversion=2)
        iinfo = r['query']['pages'][0]['imageinfo'][0]
        thumburl = iinfo['thumburl']
        # print(thumburl)
        return thumburl

def download_thumb_file(page: mwclient.page.Page) -> str:
    page_num = int(page.name.split('/')[-1])
    clean_filename = page.name.split('/')[0].split(':')[-1] + f"_page{page_num}"
    thumburl = get_thumb_url(page)

    temp_path = f"/tmp/{clean_filename}"
    with open(temp_path, "wb") as f:
        # download with user-agent
        f.write(requests.get(thumburl, headers={'User-Agent': user_agent}).content)
#        print(f"Saved image to {temp_path}")

    return temp_path

CATEGORY = "検証済"
from pathlib import Path

def get_pages_from_category(cat_name: str):
    # if category_cache.txt is available
    cache_file = "category_cache.txt"
    if not Path(cache_file).exists():
        cat = site.categories[cat_name]
        data = ""
        for page in cat:
            data += page.name + "\n"
        with open(cache_file, "w", encoding="utf-8") as f:
            f.write(data)

    with open(cache_file, "r", encoding="utf-8") as f:
        for line in f:
            yield site.pages[line.strip()]
    return

def get_text(page: mwclient.page.Page):
    return page.text()

for page in get_pages_from_category(CATEGORY):
    print(page.name)
    text = get_text(page)
    print(text[:200])
#    print(get_thumb_url(page))
    print(download_thumb_file(page))
    exit()