こちらにあるデータを使いました。
新型コロナワクチンの接種状況
データの取得
wget https://data.vrs.digital.go.jp/vaccination/opendata/latest/prefecture.ndjson
elastic_cloud_insert_ndjson.py
#! /usr/bin/python
#
# elastic_cloud_insert_ndjson.py
#
# Apr/04/2022
# ------------------------------------------------------------------
import sys
import json
import ndjson
import requests
# ------------------------------------------------------------------
def file_to_str_proc(file_in):
str_out = ""
try:
fp_in = open(file_in,encoding='utf-8')
str_out = fp_in.read()
fp_in.close()
except Exception as ee:
sys.stderr.write("*** error *** file_to_str_proc ***\n")
sys.stderr.write(str (ee))
#
return str_out
# ------------------------------------------------------------------
def elastic_insert_proc_exec(array_pp):
sys.stderr.write("*** elastic_insert_proc_exec %d ***\n" % len(array_pp))
json_str = json.dumps(array_pp)
try:
rr=requests.post(url, headers=headers, data=json_str.encode('utf-8'))
print(rr)
except Exception as ee:
sys.stderr.write("*** error *** in requests.post ***\n")
sys.stderr.write(str(ee) + "\n")
#
# ------------------------------------------------------------------
def elastic_insert_proc(array_aa):
sys.stderr.write("*** elastic_insert_proc %d ***\n" % len(array_aa))
istart = 0
while (istart < len(array_aa)):
iend = istart + 100
sys.stderr.write("*** %d\t%d ***\n" % (istart,iend))
array_p100 = array_aa[slice(istart,iend)]
elastic_insert_proc_exec(array_p100)
istart += 100
#
# ------------------------------------------------------------------
host="https://mar29-project.ent.asia-northeast1.gcp.cloud.es.io"
private_key="private-*******"
name_engine="vaccine-data-apr04"
#
sys.stderr.write("*** 開始 ***\n")
file_json = sys.argv[1]
sys.stderr.write(file_json + "\n")
#
str_in = file_to_str_proc(file_json)
#
array_aa = ndjson.loads(str_in)
#
url=host + "/api/as/v1/engines/" + name_engine + "/documents"
headers = {"Content-Type": "application/json",
"Authorization": "Bearer " + private_key}
elastic_insert_proc(array_aa)
sys.stderr.write("*** 終了 ***\n")
# ------------------------------------------------------------------
実行スクリプト
./elastic_cloud_insert_ndjson.py prefecture.ndjson