X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=tasks.py;h=29f1e5a783024a3948867d2ffc0dae8c71b09954;hb=2be22c9825d8aebe8d184e9501355a31318abf2b;hp=d680951b4090c2df8858b07786cc6292232b018a;hpb=21ed4aa91d0f1ac87ec684d8808e5ced552ad457;p=picoclvr.git diff --git a/tasks.py b/tasks.py index d680951..29f1e5a 100755 --- a/tasks.py +++ b/tasks.py @@ -27,6 +27,7 @@ def masked_inplace_autoregression( ar_mask, deterministic_synthesis, forbidden_tokens=None, + logit_biases=None, progress_bar_desc="autoregression", device=torch.device("cpu"), ): @@ -48,7 +49,11 @@ def masked_inplace_autoregression( for input, ar_mask in batches: model.masked_inplace_autoregression( - input, ar_mask, forbidden_tokens, deterministic_synthesis + input, + ar_mask, + deterministic_synthesis, + forbidden_tokens, + logit_biases, ) model.train(t) @@ -1861,3 +1866,176 @@ class QMLP(Task): ###################################################################### + +import escape + + +class Escape(Task): + def __init__( + self, + nb_train_samples, + nb_test_samples, + batch_size, + height, + width, + T, + nb_walls, + logger=None, + device=torch.device("cpu"), + ): + super().__init__() + + self.batch_size = batch_size + self.device = device + self.height = height + self.width = width + + states, actions, rewards = escape.generate_episodes( + nb_train_samples + nb_test_samples, height, width, T, nb_walls + ) + seq = escape.episodes2seq(states, actions, rewards, lookahead_delta=T) + # seq = seq[:, seq.size(1) // 3 : 2 * seq.size(1) // 3] + self.train_input = seq[:nb_train_samples].to(self.device) + self.test_input = seq[nb_train_samples:].to(self.device) + + self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1 + + def batches(self, split="train", nb_to_use=-1, desc=None): + assert split in {"train", "test"} + input = self.train_input if split == "train" else self.test_input + if nb_to_use > 0: + input = input[:nb_to_use] + if desc is None: + desc = f"epoch-{split}" + for batch in tqdm.tqdm( + input.split(self.batch_size), dynamic_ncols=True, desc=desc + ): + yield batch + + def vocabulary_size(self): + return self.nb_codes + + def thinking_autoregression( + self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 + ): + result = self.test_input[:250].clone() + t = torch.arange(result.size(1), device=result.device)[None, :] + + state_len = self.height * self.width + index_action = state_len + index_reward = state_len + 1 + index_lookahead_reward = state_len + 2 + it_len = state_len + 3 # state / action / reward / lookahead_reward + + def ar(result, ar_mask, logit_biases=None): + ar_mask = ar_mask.expand_as(result) + result *= 1 - ar_mask + masked_inplace_autoregression( + model, + self.batch_size, + result, + ar_mask, + deterministic_synthesis=deterministic_synthesis, + logit_biases=logit_biases, + device=self.device, + progress_bar_desc=None, + ) + + # Generate iteration after iteration + + optimistic_bias = result.new_zeros(self.nb_codes, device=result.device) + optimistic_bias[escape.lookahead_reward2code(-1)] = -math.log(1e1) + optimistic_bias[escape.lookahead_reward2code(1)] = math.log(1e1) + + for u in tqdm.tqdm( + range(it_len, result.size(1) - it_len + 1, it_len), desc="thinking" + ): + # Re-generate the lookahead_reward pessimistically in the + # previous iterations + ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long() + ar(result, ar_mask, logit_biases=-optimistic_bias) + + # Generate the state + ar_mask = (t >= u).long() * (t < u + state_len).long() + ar(result, ar_mask) + + # Re-generate the lookahead_reward optimistically in the + # previous iterations + ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long() + ar(result, ar_mask, logit_biases=optimistic_bias) + + # Generate the action and reward + ar_mask = (t >= u + index_action).long() * (t <= u + index_reward).long() + ar(result, ar_mask) + + # Saving the generated sequences + + s, a, r, lr = escape.seq2episodes( + result, self.height, self.width, lookahead=True + ) + str = escape.episodes2str( + s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True + ) + + filename = os.path.join(result_dir, f"test_thinking_seq_{n_epoch:04d}.txt") + with open(filename, "w") as f: + f.write(str) + logger(f"wrote {filename}") + + def produce_results( + self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 + ): + result = self.test_input[:250].clone() + + # Saving the ground truth + + s, a, r, lr = escape.seq2episodes( + result, self.height, self.width, lookahead=True + ) + str = escape.episodes2str( + s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True + ) + + filename = os.path.join(result_dir, f"test_true_seq_{n_epoch:04d}.txt") + with open(filename, "w") as f: + f.write(str) + logger(f"wrote {filename}") + + # Re-generating from the first frame + + ar_mask = ( + torch.arange(result.size(1), device=result.device) + >= self.height * self.width + 3 + ).long()[None, :] + ar_mask = ar_mask.expand_as(result) + result *= 1 - ar_mask # paraaaaanoiaaaaaaa + + masked_inplace_autoregression( + model, + self.batch_size, + result, + ar_mask, + deterministic_synthesis, + device=self.device, + ) + + # Saving the generated sequences + + s, a, r, lr = escape.seq2episodes( + result, self.height, self.width, lookahead=True + ) + str = escape.episodes2str( + s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True + ) + + filename = os.path.join(result_dir, f"test_seq_{n_epoch:04d}.txt") + with open(filename, "w") as f: + f.write(str) + logger(f"wrote {filename}") + + self.thinking_autoregression( + n_epoch, model, result_dir, logger, deterministic_synthesis, nmax + ) + + +######################################################################