使用 DQN 构建国际象棋代理

import randomclass EnvCust:    def __init__(self):        self.env = make("chess", debug=True)        self.game = game(self.env.state[0]['observation']['board'])        self.action_space = list(self.game.get_moves())        self.obs_space = fen_to_board(self.env.state[0]['observation']['board'])    def get_action(self):        return list(self.game.get_moves())    def get_obs_space(self):        return fen_to_board(self.env.state[0]['observation']['board'])    def step(self, action):        reward = 0        g = game(self.env.state[0]['observation']['board'])        if g.board.get_piece(game.xy2i(action[2:4])) == 'q':            reward = 7        elif g.board.get_piece(game.xy2i(action[2:4])) in ('n', 'b', 'r'):            reward = 4        elif g.board.get_piece(game.xy2i(action[2:4])) == 'p':            reward = 2        g.apply_move(action)        done = False        if g.status == 2:            done = True            reward = 10        elif g.status == 1:            done = True            reward = -5        self.env.step([action, 'none'])        self.action_space = self.get_action()        if not self.action_space:            done = True        else:            self.env.step(['none', random.choice(self.action_space)])            g = game(self.env.state[0]['observation']['board'])            if g.status == 2:                reward = -10                done = True        self.action_space = self.get_action()        return self.env.state[0]['observation']['board'], reward, done

import torchimport torch.nn as nnimport torch.optim as optimclass DQN(nn.Module):    def __init__(self):        super(DQN, self).__init__()        self.conv_layers = nn.Sequential(            nn.Conv2d(12, 32, kernel_size=3, stride=1, padding=1),            nn.ReLU(),            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),            nn.ReLU()        )        self.fc_layers = nn.Sequential(            nn.Flatten(),            nn.Linear(64 * 8 * 8, 256),            nn.ReLU(),            nn.Linear(256, 128),            nn.ReLU(),            nn.Linear(128, 4096)        )    def forward(self, x):        x = x.unsqueeze(0)        x = self.conv_layers(x)        x = self.fc_layers(x)        return x    def predict(self, state, valid_action_indices):        with torch.no_grad():            q_values = self.forward(state)            q_values = q_values.squeeze(0)            valid_q_values = q_values[valid_action_indices]            best_action_relative_index = valid_q_values.argmax().item()            best_action_index = valid_action_indices[best_action_relative_index]            return valid_q_values[best_action_relative_index], best_action_index

# ... (假设ReplayBuffer, fen_to_board, uci_to_action_index等函数已定义) ...device = torch.device("cuda" if torch.cuda.is_available() else "cpu")model = DQN().to(device)target_network = DQN().to(device)optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)replay_buffer = ReplayBuffer(buffer_size=10000)epsilon = 0.5gamma = 0.99batch_size = 15def train(episodes):    for ep in range(1, episodes + 1):        print('Episode Number:', ep)        myenv = EnvCust()        done = False        state = myenv.get_obs_space()        i = 0        while not done and i < batch_size:            # ... (action selection and step logic) ...            # ... (replay buffer update) ...            i += 1        if ep % 5 == 0:            target_network.load_state_dict(model.state_dict())