From: François Fleuret Date: Mon, 25 Mar 2024 16:32:05 +0000 (+0100) Subject: Update. X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=1eeba5d817d6e440a93895d42f6e580e9ba273fd;p=picoclvr.git Update. --- diff --git a/escape.py b/escape.py index f2a1662..8066479 100755 --- a/escape.py +++ b/escape.py @@ -266,11 +266,11 @@ def episodes2str( ###################################################################### if __name__ == "__main__": - nb, height, width, T, nb_walls = 5, 5, 7, 20, 5 + nb, height, width, T, nb_walls = 5, 5, 7, 4, 5 states, actions, rewards = generate_episodes(nb, height, width, T, nb_walls) seq = episodes2seq(states, actions, rewards) lr, s, a, r = seq2episodes(seq, height, width) print(episodes2str(lr, s, a, r, unicode=True, ansi_colors=True)) - # print() - # for s in seq2str(seq): - # print(s) + print() + for s in seq2str(seq): + print(s) diff --git a/tasks.py b/tasks.py index 12c6125..f2b7709 100755 --- a/tasks.py +++ b/tasks.py @@ -1893,7 +1893,7 @@ class Escape(Task): states, actions, rewards = escape.generate_episodes( nb_train_samples + nb_test_samples, height, width, T, nb_walls ) - seq = escape.episodes2seq(states, actions, rewards, lookahead_delta=T) + seq = escape.episodes2seq(states, actions, rewards) # seq = seq[:, seq.size(1) // 3 : 2 * seq.size(1) // 3] self.train_input = seq[:nb_train_samples].to(self.device) self.test_input = seq[nb_train_samples:].to(self.device) @@ -1920,10 +1920,11 @@ class Escape(Task): t = torch.arange(result.size(1), device=result.device)[None, :] state_len = self.height * self.width - index_action = state_len - index_reward = state_len + 1 - index_lookahead_reward = state_len + 2 - it_len = state_len + 3 # state / action / reward / lookahead_reward + index_lookahead_reward = 0 + index_states = 1 + index_action = state_len + 1 + index_reward = state_len + 2 + it_len = state_len + 3 # lookahead_reward / state / action / reward result[:, it_len:] = -1 @@ -1952,20 +1953,16 @@ class Escape(Task): for u in tqdm.tqdm( range(it_len, result.size(1) - it_len + 1, it_len), desc="thinking" ): - # Re-generate the lookahead_reward pessimistically in the - # previous iterations - ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long() - ar(result, ar_mask, logit_biases=-optimistic_bias) - snapshots.append(result[:10].detach().clone()) - - # Generate the state - ar_mask = (t >= u).long() * (t < u + state_len).long() + # Generate the lookahead_reward and state + ar_mask = (t >= u + index_lookahead_reward).long() * ( + t < u + index_states + state_len + ).long() ar(result, ar_mask) snapshots.append(result[:10].detach().clone()) + backup_lookahead_reward = result[:, u + index_lookahead_reward] - # Re-generate the lookahead_reward optimistically in the - # previous iterations - ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long() + # Re-generate the lookahead_reward + ar_mask = (t == u + index_lookahead_reward).long() ar(result, ar_mask, logit_biases=optimistic_bias) snapshots.append(result[:10].detach().clone()) @@ -1974,12 +1971,14 @@ class Escape(Task): ar(result, ar_mask) snapshots.append(result[:10].detach().clone()) + result[:, u + index_lookahead_reward] = backup_lookahead_reward + filename = os.path.join(result_dir, f"test_thinking_compute_{n_epoch:04d}.txt") with open(filename, "w") as f: for n in range(10): for s in snapshots: lr, s, a, r = escape.seq2episodes( - s[n : n + 1], self.height, self.width, lookahead=True + s[n : n + 1], self.height, self.width ) str = escape.episodes2str( lr, s, a, r, unicode=True, ansi_colors=True @@ -1989,7 +1988,7 @@ class Escape(Task): # Saving the generated sequences - s, a, r, lr = escape.seq2episodes(result, self.height, self.width) + lr, s, a, r = escape.seq2episodes(result, self.height, self.width) str = escape.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True) filename = os.path.join(result_dir, f"test_thinking_seq_{n_epoch:04d}.txt") @@ -2004,7 +2003,7 @@ class Escape(Task): # Saving the ground truth - s, a, r, lr = escape.seq2episodes( + lr, s, a, r = escape.seq2episodes( result, self.height, self.width, @@ -2036,7 +2035,7 @@ class Escape(Task): # Saving the generated sequences - s, a, r, lr = escape.seq2episodes( + lr, s, a, r = escape.seq2episodes( result, self.height, self.width,