kerasで強化学習

概要

kerasで強化学習してみた。
openai-gymのcartpoleしてみた。
keras-rlは、使わない。

写真

サンプルコード

import gym
import numpy as np
import time
from collections import deque
from gym import wrappers
from tensorflow.contrib.keras.python.keras.models import Sequential
from tensorflow.contrib.keras.python.keras.layers import Dense
from tensorflow.contrib.keras.python.keras.optimizers import Adam, SGD
from tensorflow.contrib.keras.python.keras import backend as K
import tensorflow as tf
import matplotlib.pyplot as plt

class Network:
	def __init__(self, learning_rate = 0.01, state_size = 4, action_size = 2, hidden_size = 10):
		self.model = Sequential()
		self.model.add(Dense(hidden_size, activation = 'tanh', input_dim = state_size))
		self.model.add(Dense(hidden_size, activation = 'tanh'))
		self.model.add(Dense(action_size, activation = 'linear'))
		self.optimizer = Adam(lr = learning_rate)
		self.model.compile(loss = self.loss, optimizer = self.optimizer)
	def replay(self, memory, batch_size, gamma):
		inputs = np.zeros((batch_size, 4))
		targets = np.zeros((batch_size, 2))
		mini_batch = memory.sample(batch_size)
		for i, (state_b, action_b, reward_b, next_state_b) in enumerate(mini_batch):
			inputs[i : i + 1] = state_b
			target = reward_b
			if not (next_state_b == np.zeros(state_b.shape)).all(axis = 1):
				retmainQs = self.model.predict(next_state_b)[0]
				next_action = np.argmax(retmainQs)
				target = reward_b + gamma * self.model.predict(next_state_b)[0][next_action]
			targets[i] = self.model.predict(state_b)
			targets[i][action_b] = target
			self.model.fit(inputs, targets, epochs = 1, verbose = 0)
	def loss(self, y_true, y_pred):
		err = y_true - y_pred
		cond = K.abs(err) < 1.0
		L2 = 0.5 * K.square(err)
		L1 = (K.abs(err) - 0.5)
		loss = tf.where(cond, L2, L1)
		return K.mean(loss)

class Memory:
	def __init__(self, max_size = 1000):
		self.buffer = deque(maxlen = max_size)
	def add(self, experience):
		self.buffer.append(experience)
	def sample(self, batch_size):
		idx = np.random.choice(np.arange(len(self.buffer)), size = batch_size, replace = False)
		return [self.buffer[ii] for ii in idx]
	def len(self):
		return len(self.buffer)

env = gym.make('CartPole-v0')
gamma = 0.99
memory_size = 5000
mainN = Network(hidden_size = 16, learning_rate = 0.00001)
memory = Memory(max_size = memory_size)
reward_trend = []
for episode in range(299):
	env.reset()
	state, reward, done, _ = env.step(env.action_space.sample())
	state = np.reshape(state, [1, 4])
	for t in range(200):
		#env.render()
		action = 0
		epsilon = 0.001 + 0.9 / (1.0 + episode)
		if epsilon <= np.random.uniform(0, 1):
			retTargetQs = mainN.model.predict(state)[0]
			action = np.argmax(retTargetQs)
		else:
			action = np.random.choice([0, 1])
		next_state, reward, done, info = env.step(action)
		next_state = np.reshape(next_state, [1, 4])
		if done:
			next_state = np.zeros(state.shape)
			if t < 195:
				reward = -1
			else:
				reward = 1
		else:
			reward = 0
		memory.add((state, action, reward, next_state))
		state = next_state
		if (memory.len() > 32):
			mainN.replay(memory, 32, gamma)
		if done:
			reward_trend.append(t + 1)
			print ('%d Episode  %d memory %d' % (episode, t + 1, memory.len()))
			break
plt.plot(reward_trend)
plt.savefig("gym31.png")
plt.show()

以上。