【量子AIが変える医療 #13】第5章 量子強化学習 手術ロボット（後編①）学習ループ前半

e)
return pos

def step(self, action: np.ndarray) -> tuple:
    """
    1ステップの実行

    Args:
        action: 関節トルク [n_joints]
    Returns:
        (next_state, reward, done, info)
    """
    dt = 0.01
    self.joint_velocities += action * dt
    self.joint_velocities = np.clip(self.joint_velocities, -1.0, 1.0)
    self.joint_angles += self.joint_velocities * dt
    self.joint_angles = np.clip(self.joint_angles, -np.pi, np.pi)

    end_effector_pos = self._get_end_effector_position()
    self.step_count += 1

    distance = np.linalg.norm(end_effector_pos - self.target_position)
    reward = -distance * 10 - np.sum(np.abs(action)) * 0.1

    # 仮想的な危険ゾーン（重要組織）
    danger_zone = np.array([0.2, 0.0, 0.4])
    danger_distance = np.linalg.norm(end_effector_pos - danger_zone)
    if danger_distance < self.safety_radius:
        reward -= 100

    done = (distance < 0.01 or
            self.step_count >= self.max_steps or
            danger_distance < self.safety_radius * 0.5)

    info = {
        'distance_to_target': distance,
        'safety_violation': danger_distance < self.safety_radius
    }

    return self._get_state(), reward, done, info

def train_quantum_surgical_robot(
n_episodes: int = 500,
batch_size: int = 32,
gamma: float = 0.99,
tau: float = 0.005
) -> dict:
"""
量子DQNによる手術ロボット学習

Args:
    n_episodes: エピソード数
    batch_size: バッチサイズ
    gamma: 割引率
    tau: ターゲットネットワーク更新率
Returns:
    学習履歴
"""
env = SurgicalRobotEnv()
policy_net = QuantumPolicyNetwork()
target_net = QuantumPolicyNetwork()
target_net.load_state_dict(policy_net.state_dict())

optimizer = Adam(policy_net.parameters(), lr=1e-3)
replay_buffer = deque(maxlen=10000)
history = {'episode_rewards': [], 'success_rate': []}

epsilon = 1.0
successes = 0

for episode in range(n_episodes):
    state = env.reset()
    episode_reward = 0