Python urllib.requestとurllib.parseを使う

はじめに

メモ用です。

この中のジオイド高を求めるAPIを用いて、httpリクエストを行い、json形式の文字列を取得し、そこからジオイド高を取得することを行う。APIの詳細については上記のURLを参照してください。

コード

import json
import urllib.parse
import urllib.request

baseurl = 'https://vldb.gsi.go.jp/sokuchi/surveycalc/geoid/calcgh/cgi/geoidcalc.pl'
params = {
    "outputType": "json",
    "latitude": 36.104611,
    "longitude": 140.084556,
}
# 辞書をパースして outputType=json&latitude=36.104611&..という文字列に変換する。
url = f'{baseurl}?{urllib.parse.urlencode(params)}'

# Requestオブジェクトを作成
req = urllib.request.Request(url)
print('---Request---')
print(f'url: {req.full_url}')
print(f'host: {req.host}')
print(f'Method: {req.get_method()}')
print(f'selector: {req.selector}')
print()

# HTTPResponseオブジェクト=file like object(with文使用可能,read()とclose()メソッドを持つ)
# ただしres.read()はバイナリデータを読み込めるようにバイト列になっている。
# そのため文字の場合はdecodeしてstrにする。
with urllib.request.urlopen(req) as res:
    print('---Response---')
    print(f'version: {res.version}')
    print(f'status: {res.status}')
    print(f'reason: {res.reason}')
    print(f'headers: {res.getheaders()}')
    print(f'body: {res.read().decode("utf-8")}')

with urllib.request.urlopen(req) as res:
    body = json.load(res)
print(body)

output

---Request---
url: https://vldb.gsi.go.jp/sokuchi/surveycalc/geoid/calcgh/cgi/geoidcalc.pl?outputType=json&latitude=36.104611&longitude=140.084556
host: vldb.gsi.go.jp
Method: GET
selector: /sokuchi/surveycalc/geoid/calcgh/cgi/geoidcalc.pl?outputType=json&latitude=36.104611&longitude=140.084556

---Response---
version: 11
status: 200
reason: OK
headers: [('Date', 'Fri, 14 Jun 2024 06:05:28 GMT'), ('Server', 'Apache'), ('X-content-type-options', 'nosniff'), ('Strict-Transport-Security', 'max-age=15768000'), ('X-Frame-Options', 'SAMEORIGIN'), ('X-XSS-Protection', '1; mode=block'), ('X-Permitted-Cross-Domain-Policies', 'none'), ('Referrer-Policy', 'origin'), ('Content-Security-Policy', "default-src 'self' *.gsi.go.jp data:;      style-src 'self' 'unsafe-inline' *.gsi.go.jp;      script-src 'self' 'unsafe-inline' 'unsafe-eval' *.gsi.go.jp cdnjs.cloudflare.com cdn.jsdelivr.net;      upgrade-insecure-requests"), ('Access-Control-Allow-Origin', 'http://vldb.gsi.go.jp https://vldb.gsi.go.jp'), ('Cache-Control', 'no-store'), ('Connection', 'close'), ('Transfer-Encoding', 'chunked'), ('Content-Type', 'application/json; charset=UTF-8')]
body: {"OutputData":{"latitude":"36.104611000","longitude":"140.084556000","geoidHeight":"40.1986"}}
{'OutputData': {'latitude': '36.104611000', 'longitude': '140.084556000', 'geoidHeight': '40.1986'}}

同じのをrequestsで書くと

import requests

baseurl = 'https://vldb.gsi.go.jp/sokuchi/surveycalc/geoid/calcgh/cgi/geoidcalc.pl'
params = {
    "outputType": "json",
    "latitude": 36.104611,
    "longitude": 140.084556,
}

r = requests.get(baseurl, params=params)
print(r.reason)
print(r.ok)
print(r.status_code)
print(r.json())

output

OK
True
200
{'OutputData': {'latitude': '36.104611000', 'longitude': '140.084556000', 'geoidHeight': '40.1986'}}

ProxyHandler

import urllib.request

# proxyHandler
proxies = {
    'http': 'http://proxy.example.com:8080',
    'https': 'http://proxy.example.com:8080'
}
proxy_handler = urllib.request.ProxyHandler(proxies)

url = 'https://qiita.com'

# opener
opener = urllib.request.build_opener()
opener.add_handler(proxy_handler)

# using opener
req = urllib.request.Request(url)
response = opener.open(req)
print(response.read().decode('utf-8')[0])

# install_opener
urllib.request.install_opener(opener)
req = urllib.request.Request(url)
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8')[0])

ProxyBasicAuthHandler

import urllib.request

# プロキシサーバーの設定
proxy_url = 'http://proxy.example.com:8080'
proxy_username = 'your_proxy_username'
proxy_password = 'your_proxy_password'

proxy_auth_handler = urllib.request.ProxyBasicAuthHandler()
proxy_auth_handler.add_password("", proxy_url, proxy_username, proxy_password)

url = 'https://qiita.com'

# opener
opener = urllib.request.build_opener()
opener.add_handler(proxy_auth_handler)


# using opener
req = urllib.request.Request(url)
response = opener.open(req)
print(response.read().decode('utf-8')[0])

# install_opener
urllib.request.install_opener(opener)
req = urllib.request.Request(url)
response = urllib.request.urlopen(req)
print(response.read().decode('utf-8')[0])