X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=tasks.py;h=12c6125348691381a7bea80787f7da3107a45253;hb=2bb362ef385f477da4af7d8679cc94d42cf6c146;hp=38c85ed84d228b217e214b68adbbda1f76c255a9;hpb=690470307a7995cc3117eb54545f921eedcecba5;p=picoclvr.git diff --git a/tasks.py b/tasks.py index 38c85ed..12c6125 100755 --- a/tasks.py +++ b/tasks.py @@ -27,6 +27,7 @@ def masked_inplace_autoregression( ar_mask, deterministic_synthesis, forbidden_tokens=None, + logit_biases=None, progress_bar_desc="autoregression", device=torch.device("cpu"), ): @@ -48,7 +49,11 @@ def masked_inplace_autoregression( for input, ar_mask in batches: model.masked_inplace_autoregression( - input, ar_mask, forbidden_tokens, deterministic_synthesis + input, + ar_mask, + deterministic_synthesis, + forbidden_tokens, + logit_biases, ) model.train(t) @@ -1874,6 +1879,7 @@ class Escape(Task): height, width, T, + nb_walls, logger=None, device=torch.device("cpu"), ): @@ -1885,15 +1891,13 @@ class Escape(Task): self.width = width states, actions, rewards = escape.generate_episodes( - nb_train_samples + nb_test_samples, height, width, T + nb_train_samples + nb_test_samples, height, width, T, nb_walls ) seq = escape.episodes2seq(states, actions, rewards, lookahead_delta=T) # seq = seq[:, seq.size(1) // 3 : 2 * seq.size(1) // 3] self.train_input = seq[:nb_train_samples].to(self.device) self.test_input = seq[nb_train_samples:].to(self.device) - self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1 - def batches(self, split="train", nb_to_use=-1, desc=None): assert split in {"train", "test"} input = self.train_input if split == "train" else self.test_input @@ -1907,53 +1911,86 @@ class Escape(Task): yield batch def vocabulary_size(self): - return self.nb_codes + return escape.nb_codes def thinking_autoregression( self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 ): - result = self.test_input[:100].clone() - t = torch.arange(result.size(1), device=result.device) + result = self.test_input[:250].clone() + t = torch.arange(result.size(1), device=result.device)[None, :] + state_len = self.height * self.width - iteration_len = state_len + 3 + index_action = state_len + index_reward = state_len + 1 + index_lookahead_reward = state_len + 2 + it_len = state_len + 3 # state / action / reward / lookahead_reward + + result[:, it_len:] = -1 - def ar(): + def ar(result, ar_mask, logit_biases=None): + ar_mask = ar_mask.expand_as(result) + result *= 1 - ar_mask masked_inplace_autoregression( model, self.batch_size, result, ar_mask, - deterministic_synthesis, + deterministic_synthesis=deterministic_synthesis, + logit_biases=logit_biases, device=self.device, + progress_bar_desc=None, ) - for u in range( - iteration_len, result.size(1) - iteration_len + 1, iteration_len + # Generate iteration after iteration + + optimistic_bias = result.new_zeros(escape.nb_codes, device=result.device) + optimistic_bias[escape.lookahead_reward2code(-1)] = -math.log(1e1) + optimistic_bias[escape.lookahead_reward2code(1)] = math.log(1e1) + + snapshots = [] + + for u in tqdm.tqdm( + range(it_len, result.size(1) - it_len + 1, it_len), desc="thinking" ): - # Put a lookahead reward to -1, sample the next state - result[:, u - 1] = (-1) + 1 + escape.first_lookahead_rewards_code + # Re-generate the lookahead_reward pessimistically in the + # previous iterations + ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long() + ar(result, ar_mask, logit_biases=-optimistic_bias) + snapshots.append(result[:10].detach().clone()) + + # Generate the state ar_mask = (t >= u).long() * (t < u + state_len).long() - ar_mask = ar_mask[None, :] - ar_mask = ar_mask.expand_as(result) - result *= 1 - ar_mask - ar() + ar(result, ar_mask) + snapshots.append(result[:10].detach().clone()) - # Put a lookahead reward to +1, sample the action and reward - result[:, u - 1] = (1) + 1 + escape.first_lookahead_rewards_code - ar_mask = (t >= state_len).long() * (t < state_len + 2).long() - ar_mask = ar_mask[None, :] - ar_mask = ar_mask.expand_as(result) - result *= 1 - ar_mask - ar() + # Re-generate the lookahead_reward optimistically in the + # previous iterations + ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long() + ar(result, ar_mask, logit_biases=optimistic_bias) + snapshots.append(result[:10].detach().clone()) + + # Generate the action and reward + ar_mask = (t >= u + index_action).long() * (t <= u + index_reward).long() + ar(result, ar_mask) + snapshots.append(result[:10].detach().clone()) + + filename = os.path.join(result_dir, f"test_thinking_compute_{n_epoch:04d}.txt") + with open(filename, "w") as f: + for n in range(10): + for s in snapshots: + lr, s, a, r = escape.seq2episodes( + s[n : n + 1], self.height, self.width, lookahead=True + ) + str = escape.episodes2str( + lr, s, a, r, unicode=True, ansi_colors=True + ) + f.write(str) + f.write("\n\n") # Saving the generated sequences - s, a, r, lr = escape.seq2episodes( - result, self.height, self.width, lookahead=True - ) - str = escape.episodes2str( - s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True - ) + s, a, r, lr = escape.seq2episodes(result, self.height, self.width) + str = escape.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True) filename = os.path.join(result_dir, f"test_thinking_seq_{n_epoch:04d}.txt") with open(filename, "w") as f: @@ -1963,16 +2000,16 @@ class Escape(Task): def produce_results( self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 ): - result = self.test_input[:100].clone() + result = self.test_input[:250].clone() # Saving the ground truth s, a, r, lr = escape.seq2episodes( - result, self.height, self.width, lookahead=True - ) - str = escape.episodes2str( - s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True + result, + self.height, + self.width, ) + str = escape.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True) filename = os.path.join(result_dir, f"test_true_seq_{n_epoch:04d}.txt") with open(filename, "w") as f: @@ -2000,11 +2037,11 @@ class Escape(Task): # Saving the generated sequences s, a, r, lr = escape.seq2episodes( - result, self.height, self.width, lookahead=True - ) - str = escape.episodes2str( - s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True + result, + self.height, + self.width, ) + str = escape.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True) filename = os.path.join(result_dir, f"test_seq_{n_epoch:04d}.txt") with open(filename, "w") as f: