グラフ構造を深層学習する PyG (PyTorch Geometric) を Google Colaboratory 上で使ってみました。今回は、Recurrent Graph Neural Network を使うことがテーマです。
学習済みモデルを Google Drive に保存するための準備
学習済みモデルを Google Drive に保存するため、Google Drive をマウントします。
from google.colab import drive
drive.mount('/content/drive')
Mounted at /content/drive
次のようにして Google Drive 上のディレクトリを指定します。
import os
directory_path = './drive/MyDrive/pyg_models/'
if not os.path.exists(directory_path):
os.makedirs(directory_path)
PyG (PyTorch Geometric) インストール
PyG (PyTorch Geometric) のレポジトリは https://github.com/pyg-team/pytorch_geometric にあります。また、コードはチュートリアルドキュメント https://pytorch-geometric.readthedocs.io/en/latest/index.html を参考にしています。
import os
import torch
torch.manual_seed(53)
os.environ['TORCH'] = torch.__version__
print(torch.__version__)
!pip install -q torch-scatter -f https://data.pyg.org/whl/torch-${TORCH}.html
!pip install -q torch-sparse -f https://data.pyg.org/whl/torch-${TORCH}.html
!pip install -q git+https://github.com/pyg-team/pytorch_geometric.git
import torch_geometric
1.12.0+cu113
[K |████████████████████████████████| 7.9 MB 45.1 MB/s
[K |████████████████████████████████| 3.5 MB 24.4 MB/s
[?25h Building wheel for torch-geometric (setup.py) ... [?25l[?25hdone
コード開発中は cpu を用い、max_epoch を 50 程度にしていました。ほぼ開発が終わってから、cuda を用いて max_epoch を 2000 にしました。
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
#device = "cpu"
max_epoch = 2000
次のようにして、train_mask, val_mask, test_mask を設定します。
def train_val_test_split(data, val_ratio: float = 0.15,
test_ratio: float = 0.15):
rnd = torch.rand(len(data.x))
train_mask = [False if (x > val_ratio + test_ratio) else True for x in rnd]
val_mask = [False if (val_ratio + test_ratio >= x) and (x > test_ratio) else True for x in rnd]
test_mask = [False if (test_ratio >= x) else True for x in rnd]
return torch.tensor(train_mask), torch.tensor(val_mask), torch.tensor(test_mask)
日本の市町村の緯度経度データ
日本の市町村の緯度経度データを PyG で取り扱うためのクラスです。コードの内容について詳細は過去記事をご参照ください。
import numpy as np
import pandas as pd
from scipy.spatial import distance
from torch_geometric.data import Data, InMemoryDataset
class JapaneseCities(InMemoryDataset):
def __init__(self, transform = None,
top=8,
url="https://raw.githubusercontent.com/maskot1977/toydata/main/data/cities3.txt"):
super().__init__('.', transform)
japan_df = pd.read_csv(url, header=None, sep="\t")
location = japan_df.iloc[:, -2:].values
dist_matrix = distance.cdist(location, location, metric='euclidean')
adj_matrix = []
for idxs in np.argsort(dist_matrix)[:, :top]:
adj_matrix.append(
[1 if i in idxs else 0 for i in range(len(dist_matrix))]
)
adj_matrix = np.array(adj_matrix)
embeddings = torch.tensor(japan_df.iloc[:, [-1, -2]].values,
dtype=torch.float)
edges = []
edge_attr = []
for i in range(len(adj_matrix)):
for j in range(len(adj_matrix)):
if i == j:
continue
elif adj_matrix[i][j] == 1:
edges.append([i, j])
edge_attr.append(dist_matrix[i][j])
edges = torch.tensor(edges, dtype=torch.long).T
edge_attr = torch.tensor(edge_attr, dtype=torch.float)
name2id = {}
id2name = {}
for v in japan_df.iloc[:, 1].values:
if v not in name2id.keys():
name2id[v] = len(name2id.keys())
id2name[name2id[v]] = v
ys = torch.tensor(
[name2id[v] for v in japan_df.iloc[:, 1].values], dtype=torch.long
)
data = Data(x=embeddings, edge_index=edges, y=ys, edge_attr=edge_attr)
self.data, self.slices = self.collate([data])
次のようにしてデータセットを作成します。この data はのちの工程でグローバル変数として用いられます。
dataset = JapaneseCities()
data = dataset[0].to(device)
train_mask, val_mask, test_mask = train_val_test_split(data)
data.train_mask = train_mask
data.val_mask = val_mask
data.test_mask = test_mask
学習の流れ
過去記事とは違う書き方で学習してみたいと思います。model や data は引数として与えていないので、グローバル変数から取ってくるということに注意してください。
def train():
model.train()
optimizer.zero_grad()
loss = loss_fn(model()[data.train_mask], data.y[data.train_mask])
loss.backward()
optimizer.step()
return loss.detach().cpu().numpy()
def test():
model.eval()
logits = model()
accs = []
for _, mask in data('train_mask', 'val_mask', 'test_mask'):
pred = logits[mask].max(1)[1]
acc = pred.eq(data.y[mask]).sum().item() / mask.sum().item()
accs.append(acc)
return accs
Graph Attention Networks (GAT)
過去記事でも用いた Graph Attention Networks (GAT) (これは Recurrent Graph Neural Network ではありません)を今回は次のように定義します。forward の引数が self だけであることに注意してください。
class GAT(torch.nn.Module):
def __init__(self):
super(GAT, self).__init__()
self.hid = 32
self.in_head = 32
self.out_head = 1
self.conv1 = torch_geometric.nn.GATConv(dataset.num_features,
self.hid,
heads=self.in_head,
dropout=0.1) # dropout=0.6)
self.conv1b = torch_geometric.nn.GATConv(self.hid*self.in_head,
self.hid,
heads=self.in_head,
dropout=0.1)
self.conv2 = torch_geometric.nn.GATConv(self.hid*self.in_head,
dataset.num_classes,
heads=self.out_head,
concat=True,
dropout=0.1)
def forward(self):
x, edge_index = data.x, data.edge_index
x = torch.nn.functional.dropout(x, p=0.1, training=self.training)
x = self.conv1(x, edge_index)
x = torch.nn.functional.elu(x)
x = torch.nn.functional.dropout(x, p=0.1, training=self.training)
x = self.conv1b(x, edge_index)
x = torch.nn.functional.elu(x)
x = torch.nn.functional.dropout(x, p=0.1, training=self.training)
x = self.conv2(x, edge_index)
return torch.nn.functional.log_softmax(x, dim=1)
モデル、最適化手法、損失関数を定義します。
model_name = "GAT"
model = GAT().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
loss_fn = torch.nn.CrossEntropyLoss()
次のようにして学習します。このコードは使いまわします。
import copy
best_loss = None
loss_hist = []
acc_hist = []
for epoch in range(1, max_epoch):
loss = train()
loss_hist.append(loss)
accs = test()
acc_hist.append(accs)
train_acc, val_acc, test_acc = accs
if best_loss is None or best_loss > loss:
best_loss = loss
best_model = copy.deepcopy(model)
best_model = best_model.to('cpu')
torch.save(
best_model.state_dict(),
"{}/{}_model.pt".format(directory_path, model_name)
)
print('Epoch: {:05d}, Train Acc: {:.5f}, '
'Val Acc: {:.5f}, Test Acc: {:.5f}, Loss: {}'.format(
epoch, train_acc, val_acc, test_acc, loss))
Epoch: 00001, Train Acc: 0.18740, Val Acc: 0.18742, Test Acc: 0.18700, Loss: 12.414194107055664
Epoch: 00002, Train Acc: 0.12604, Val Acc: 0.11817, Test Acc: 0.12384, Loss: 8.90912914276123
Epoch: 00019, Train Acc: 0.14760, Val Acc: 0.12770, Test Acc: 0.12879, Loss: 8.612857818603516
Epoch: 00020, Train Acc: 0.18740, Val Acc: 0.18742, Test Acc: 0.18700, Loss: 7.044679164886475
Epoch: 00022, Train Acc: 0.29685, Val Acc: 0.28971, Test Acc: 0.29412, Loss: 6.814143180847168
(以下略)
損失関数や正答率の履歴を表示してみましょう。
import matplotlib.pyplot as plt
plt.title(model_name)
plt.plot(loss_hist, label="Loss")
plt.legend()
plt.yscale('log')
plt.grid()
plt.show()
plt.title(model_name)
plt.plot([acc[0] for acc in acc_hist], label="Train Acc")
plt.plot([acc[1] for acc in acc_hist], label="Val Acc")
plt.plot([acc[2] for acc in acc_hist], label="Test Acc")
plt.legend()
plt.grid()
plt.show()
学習が進むにつれて、損失関数の値は低下していってますが、正答率はほとんど向上していないようです。
Graph Neural Network Model (GNNM)
これは graph convolution に MessagePassing と MLP を導入して改良したネットワークモデルです。
class MLP(torch.nn.Module):
def __init__(self, input_dim, hid_dims, out_dim):
super(MLP, self).__init__()
self.mlp = torch.nn.Sequential()
dims = [input_dim] + hid_dims + [out_dim]
for i in range(len(dims)-1):
self.mlp.add_module(
'lay_{}'.format(i),
torch.nn.Linear(in_features=dims[i], out_features=dims[i+1])
)
if i+2 < len(dims):
self.mlp.add_module('act_{}'.format(i), torch.nn.Tanh())
def reset_parameters(self):
for i, l in enumerate(self.mlp):
if type(l) == torch.nn.Linear:
torch.nn.init.xavier_normal_(l.weight)
def forward(self, x):
return self.mlp(x)
class GNNM(torch_geometric.nn.conv.MessagePassing):
def __init__(self, n_nodes, out_channels, features_dim, hid_dims,
num_layers = 50, eps=1e-3, aggr = 'add',
bias = True, **kwargs):
super(GNNM, self).__init__(aggr=aggr, **kwargs)
self.node_states = torch.nn.Parameter(
torch.zeros((n_nodes, features_dim)), requires_grad=False
)
self.out_channels = out_channels
self.eps = eps
self.num_layers = num_layers
self.transition = MLP(features_dim, hid_dims, features_dim)
self.readout = MLP(features_dim, hid_dims, out_channels)
self.reset_parameters()
def reset_parameters(self):
self.transition.reset_parameters()
self.readout.reset_parameters()
def forward(self):
edge_index = data.edge_index
edge_weight = data.edge_attr
node_states = self.node_states
for i in range(self.num_layers):
m = self.propagate(
edge_index, x=node_states, edge_weight=edge_weight,
size=None)
new_states = self.transition(m)
with torch.no_grad():
distance = torch.norm(new_states - node_states, dim=1)
convergence = distance < self.eps
node_states = new_states
if convergence.all():
break
out = self.readout(node_states)
return torch.nn.functional.log_softmax(out, dim=-1)
def message(self, x_j, edge_weight):
return x_j if edge_weight is None else edge_weight.view(-1, 1) * x_j
def message_and_aggregate(self, adj_t, x) :
return matmul(adj_t, x, reduce=self.aggr)
def __repr__(self):
return '{}({}, num_layers={})'.format(self.__class__.__name__,
self.out_channels,
self.num_layers)
モデル、最適化手法、損失関数を定義します。
model_name = "GNNM"
model = GNNM(
data.num_nodes, dataset.num_classes, 32, [64,64,64,64,64], eps=0.01
).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
loss_fn = torch.nn.CrossEntropyLoss()
次のようにして学習します。このコードは先程の使いまわしです。
import copy
best_loss = None
loss_hist = []
acc_hist = []
for epoch in range(1, max_epoch):
loss = train()
loss_hist.append(loss)
accs = test()
acc_hist.append(accs)
train_acc, val_acc, test_acc = accs
if best_loss is None or best_loss > loss:
best_loss = loss
best_model = copy.deepcopy(model)
best_model = best_model.to('cpu')
torch.save(
best_model.state_dict(),
"{}/{}_model.pt".format(directory_path, model_name)
)
print('Epoch: {:05d}, Train Acc: {:.5f}, '
'Val Acc: {:.5f}, Test Acc: {:.5f}, Loss: {}'.format(
epoch, train_acc, val_acc, test_acc, loss))
Epoch: 00001, Train Acc: 0.16086, Val Acc: 0.15756, Test Acc: 0.16409, Loss: 2.4063360691070557
Epoch: 00002, Train Acc: 0.17745, Val Acc: 0.17027, Test Acc: 0.17399, Loss: 2.3022396564483643
Epoch: 00003, Train Acc: 0.21891, Val Acc: 0.18297, Test Acc: 0.19071, Loss: 2.25955867767334
Epoch: 00004, Train Acc: 0.22720, Val Acc: 0.21537, Test Acc: 0.21176, Loss: 2.222214460372925
Epoch: 00005, Train Acc: 0.18740, Val Acc: 0.18679, Test Acc: 0.18638, Loss: 2.203711986541748
(以下略)
損失関数や正答率の履歴を表示してみましょう。
import matplotlib.pyplot as plt
plt.title(model_name)
plt.plot(loss_hist, label="Loss")
plt.legend()
plt.yscale('log')
plt.grid()
plt.show()
plt.title(model_name)
plt.plot([acc[0] for acc in acc_hist], label="Train Acc")
plt.plot([acc[1] for acc in acc_hist], label="Val Acc")
plt.plot([acc[2] for acc in acc_hist], label="Test Acc")
plt.legend()
plt.grid()
plt.show()
学習が収束してない気がしますが、先程のGATよりは良い成績が出てます。
Gated Graph Neural Network (GGNN)
こちらは graph convolution の MessagePassing に GRU (gated recurrent unit) を導入したネットワークです。
class GatedGraphConv(torch_geometric.nn.conv.MessagePassing):
def __init__(self, out_channels, num_layers, aggr = 'add',
bias = True, **kwargs):
super(GatedGraphConv, self).__init__(aggr=aggr, **kwargs)
self.out_channels = out_channels
self.num_layers = num_layers
self.weight = torch.nn.Parameter(torch.Tensor(num_layers, out_channels, out_channels))
self.rnn = torch.nn.GRUCell(out_channels, out_channels, bias=bias)
self.reset_parameters()
def reset_parameters(self):
torch_geometric.nn.inits.uniform(self.out_channels, self.weight)
self.rnn.reset_parameters()
def forward(self, data):
x = data.x
edge_index = data.edge_index
edge_weight = data.edge_attr
if x.size(-1) > self.out_channels:
raise ValueError('The number of input channels is not allowed to '
'be larger than the number of output channels')
if x.size(-1) < self.out_channels:
zero = x.new_zeros(x.size(0), self.out_channels - x.size(-1))
x = torch.cat([x, zero], dim=1)
for i in range(self.num_layers):
m = torch.matmul(x, self.weight[i])
m = self.propagate(edge_index, x=m, edge_weight=edge_weight,
size=None)
x = self.rnn(m, x)
return x
def message(self, x_j, edge_weight):
return x_j if edge_weight is None else edge_weight.view(-1, 1) * x_j
def message_and_aggregate(self, adj_t, x):
return matmul(adj_t, x, reduce=self.aggr)
def __repr__(self):
return '{}({}, num_layers={})'.format(self.__class__.__name__,
self.out_channels,
self.num_layers)
class GGNN(torch.nn.Module):
def __init__(self):
super(GGNN, self).__init__()
self.conv = GatedGraphConv(1433, 3)
self.mlp = MLP(1433, [32,32,32], dataset.num_classes)
def forward(self):
x = self.conv(data)
x = self.mlp(x)
return torch.nn.functional.log_softmax(x, dim=-1)
モデル、最適化手法、損失関数を定義します。
model_name = "GGNN"
model = GGNN().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
loss_fn = torch.nn.CrossEntropyLoss()
次のようにして学習します。このコードは先程の使いまわしです。
import copy
best_loss = None
loss_hist = []
acc_hist = []
for epoch in range(1, max_epoch):
loss = train()
loss_hist.append(loss)
accs = test()
acc_hist.append(accs)
train_acc, val_acc, test_acc = accs
if best_loss is None or best_loss > loss:
best_loss = loss
best_model = copy.deepcopy(model)
best_model = best_model.to('cpu')
torch.save(
best_model.state_dict(),
"{}/{}_model.pt".format(directory_path, model_name)
)
print('Epoch: {:05d}, Train Acc: {:.5f}, '
'Val Acc: {:.5f}, Test Acc: {:.5f}, Loss: {}'.format(
epoch, train_acc, val_acc, test_acc, loss))
Epoch: 00001, Train Acc: 0.18242, Val Acc: 0.17408, Test Acc: 0.17833, Loss: 2.4117162227630615
Epoch: 00002, Train Acc: 0.23051, Val Acc: 0.20457, Test Acc: 0.21610, Loss: 2.334713935852051
Epoch: 00003, Train Acc: 0.23715, Val Acc: 0.22935, Test Acc: 0.22910, Loss: 2.3105783462524414
Epoch: 00004, Train Acc: 0.23715, Val Acc: 0.22490, Test Acc: 0.22972, Loss: 2.279224395751953
Epoch: 00005, Train Acc: 0.20232, Val Acc: 0.19886, Test Acc: 0.19876, Loss: 2.258718252182007
(以下略)
損失関数や正答率の履歴を表示してみましょう。
import matplotlib.pyplot as plt
plt.title(model_name)
plt.plot(loss_hist, label="Loss")
plt.legend()
plt.yscale('log')
plt.grid()
plt.show()
plt.title(model_name)
plt.plot([acc[0] for acc in acc_hist], label="Train Acc")
plt.plot([acc[1] for acc in acc_hist], label="Val Acc")
plt.plot([acc[2] for acc in acc_hist], label="Test Acc")
plt.legend()
plt.grid()
plt.show()
今まで試したどのモデルよりも早く収束し、損失関数はまだまだ下がりそうですが、予測性能は Train ACC = 1.0 に到達し頭打ちになりました。
保存したモデルの読み出し
次のようにして保存したモデルを読み出して予測に使えます。
model_path = "{}/{}_model.pt".format(directory_path, "GAT")
gat_model = GAT().to(device)
gat_model.load_state_dict(
torch.load(model_path, map_location=torch.device('cpu'))
)
_, gat_pred = gat_model().max(dim=1)
gat_pred
tensor([ 0, 0, 0, ..., 10, 10, 10], device='cuda:0')
model_path = "{}/{}_model.pt".format(directory_path, "GNNM")
gnnm_model = GNNM(
data.num_nodes, dataset.num_classes, 32, [64,64,64,64,64], eps=0.01
).to(device)
gnnm_model.load_state_dict(
torch.load(model_path, map_location=torch.device('cpu'))
)
_, gnnm_pred = gnnm_model().max(dim=1)
gnnm_pred
tensor([ 1, 1, 0, ..., 10, 10, 10], device='cuda:0')
model_path = "{}/{}_model.pt".format(directory_path, "GGNN")
ggnn_model = GGNN().to(device)
ggnn_model.load_state_dict(
torch.load(model_path, map_location=torch.device('cpu'))
)
_, ggnn_pred = ggnn_model().max(dim=1)
ggnn_pred
tensor([ 0, 0, 0, ..., 10, 10, 10], device='cuda:0')