from bs4 import BeautifulSoup
import requests

res = requests.get(url)
soup = BeautifulSoup(res.content, 'html.parser')

from bs4 import BeautifulSoup
import requests

res = requests.get(url)
soup = BeautifulSoup(res.content, "lxml", from_encoding='utf-8')

# Seleniumを使用するのに必要なライブラリをダウンロード
!apt-get update
!apt install chromium-chromedriver
!cp /usr/lib/chromium-browser/chromedriver /usr/bin
!pip install selenium

from selenium import webdriver

# ドライバーをUIなしで使用するための設定
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('chromedriver',options=options)
driver.implicitly_wait(10)

driver.get(url)
html = driver.page_source.encode('utf-8')
soup = BeautifulSoup(html, 'html.parser')

from selenium.webdriver.support.ui import Select

element = driver.find_element_by_xpath("xpath")
Select(element).select_by_index(indexnum) # indexで選択
Select(element).select_by_value("value") # valueの値
Select(element).select_by_visible_text("text") # 表示テキスト

handle_array = driver.window_handles
driver.switch_to.window(handle_array[1])

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

WebDriverWait(driver, 30).until(
  EC.presence_of_element_located((By.XPATH, 'xpath')))
)

target = driver.find_element_by_xpath('xpath')
driver.execute_script("arguments[0].click();", target)

import pandas as pd
columns = ['項目1', '項目2', '項目3', '項目4', '項目5']
df = pd.DataFrame(columns=columns)

# データ取得の処理

se = pd.Series([data1, data2, data3, data4, data5], columns)
df = df.append(se, columns)

from google.colab import files

filename = 'filename.csv'
df.to_csv(filename, encoding = 'utf-8-sig') 
files.download(filename)

from google.colab import drive

filename = filename.csv'
path = '/content/drive/My Drive/' + filename

with open(path, 'w', encoding = 'utf-8-sig') as f:
  df.to_csv(f)

# スプレッドシート連携するのに必要なライブラリをダウンロード
!pip install gspread

from google.colab import auth
from oauth2client.client import GoogleCredentials
import gspread

# 認証処理
auth.authenticate_user()
gc = gspread.authorize(GoogleCredentials.get_application_default())

ss_id = 'スプレッドシートのID'
sht_name = 'シート名'

workbook = gc.open_by_key(ss_id)
worksheet = workbook.worksheet(sht_name)

# データ取得するとき
worksheet.acell('B1').value
worksheet.cell(2, 1).value

# アップデートするとき
worksheet.update_cell(row, column, '更新内容')