X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=tasks.py;h=324376df60319e9549ae431c5d43dd04f1a29ed9;hb=232299b8af7e66a02e64bb2e47b525e2f50b099d;hp=51538366383be455f160ab6158392627ecea5190;hpb=19ec7f3e4030ddece2647983dcf1bed5eb0d9544;p=picoclvr.git diff --git a/tasks.py b/tasks.py index 5153836..324376d 100755 --- a/tasks.py +++ b/tasks.py @@ -5,7 +5,7 @@ # Written by Francois Fleuret -import math, os, tqdm +import math, os, tqdm, warnings import torch, torchvision @@ -27,6 +27,7 @@ def masked_inplace_autoregression( ar_mask, deterministic_synthesis, forbidden_tokens=None, + logit_biases=None, progress_bar_desc="autoregression", device=torch.device("cpu"), ): @@ -48,7 +49,11 @@ def masked_inplace_autoregression( for input, ar_mask in batches: model.masked_inplace_autoregression( - input, ar_mask, forbidden_tokens, deterministic_synthesis + input, + ar_mask, + deterministic_synthesis, + forbidden_tokens, + logit_biases, ) model.train(t) @@ -1862,10 +1867,10 @@ class QMLP(Task): ###################################################################### -import escape +import greed -class Escape(Task): +class Greed(Task): def __init__( self, nb_train_samples, @@ -1875,6 +1880,7 @@ class Escape(Task): width, T, nb_walls, + nb_coins, logger=None, device=torch.device("cpu"), ): @@ -1882,18 +1888,27 @@ class Escape(Task): self.batch_size = batch_size self.device = device - self.height = height - self.width = width - states, actions, rewards = escape.generate_episodes( - nb_train_samples + nb_test_samples, height, width, T, nb_walls + self.world = greed.GreedWorld(height, width, T, nb_walls, nb_coins) + + states, actions, rewards = self.world.generate_episodes( + nb_train_samples + nb_test_samples ) - seq = escape.episodes2seq(states, actions, rewards, lookahead_delta=T) - # seq = seq[:, seq.size(1) // 3 : 2 * seq.size(1) // 3] + seq = self.world.episodes2seq(states, actions, rewards) self.train_input = seq[:nb_train_samples].to(self.device) self.test_input = seq[nb_train_samples:].to(self.device) - self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1 + def wipe_lookahead_rewards(self, batch): + t = torch.arange(batch.size(1), device=batch.device)[None, :] + u = torch.randint(batch.size(1), (batch.size(0), 1), device=batch.device) + lr_mask = (t <= u).long() * ( + t % self.world.it_len == self.world.index_lookahead_reward + ).long() + + return ( + lr_mask * self.world.lookahead_reward2code(greed.REWARD_UNKNOWN) + + (1 - lr_mask) * batch + ) def batches(self, split="train", nb_to_use=-1, desc=None): assert split in {"train", "test"} @@ -1905,21 +1920,17 @@ class Escape(Task): for batch in tqdm.tqdm( input.split(self.batch_size), dynamic_ncols=True, desc=desc ): - yield batch + yield self.wipe_lookahead_rewards(batch) def vocabulary_size(self): - return self.nb_codes + return self.world.nb_codes def thinking_autoregression( self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 ): - result = self.test_input[:250].clone() - t = torch.arange(result.size(1), device=result.device)[None, :] + snapshots = [] - state_len = self.height * self.width - it_len = state_len + 3 # state / action / reward / lookahead_reward - - def ar(result, ar_mask): + def ar(result, ar_mask, logit_biases=None): ar_mask = ar_mask.expand_as(result) result *= 1 - ar_mask masked_inplace_autoregression( @@ -1927,56 +1938,73 @@ class Escape(Task): self.batch_size, result, ar_mask, - deterministic_synthesis, + deterministic_synthesis=deterministic_synthesis, + logit_biases=logit_biases, device=self.device, progress_bar_desc=None, ) + warnings.warn("keeping thinking snapshots", RuntimeWarning) + snapshots.append(result[:10].detach().clone()) # Generate iteration after iteration + result = self.test_input[:250].clone() + # Erase all the content but that of the first iteration + result[:, self.world.it_len :] = -1 + # Set the lookahead_reward of the firs to UNKNOWN + result[:, self.world.index_lookahead_reward] = self.world.lookahead_reward2code( + greed.REWARD_UNKNOWN + ) + + t = torch.arange(result.size(1), device=result.device)[None, :] + for u in tqdm.tqdm( - range(it_len, result.size(1) - it_len + 1, it_len), desc="thinking" + range(0, result.size(1), self.world.it_len), + desc="thinking", ): - # Put the lookahead reward to either 0 or -1 for the - # current iteration, with a proba that depends with the - # sequence index, so that we have diverse examples, sample - # the next state - s = -( - torch.rand(result.size(0), device=result.device) - <= torch.linspace(0, 1, result.size(0), device=result.device) + # Generate the next state but keep the initial one, the + # lookahead_reward of previous iterations are set to + # UNKNOWN + if u > 0: + result[ + :, u + self.world.index_lookahead_reward + ] = self.world.lookahead_reward2code(greed.REWARD_UNKNOWN) + ar_mask = (t >= u + self.world.index_states).long() * ( + t < u + self.world.index_states + self.world.state_len + ).long() + ar(result, ar_mask) + + # Generate the action and reward with lookahead_reward to +1 + result[ + :, u + self.world.index_lookahead_reward + ] = self.world.lookahead_reward2code(greed.REWARD_PLUS) + ar_mask = (t >= u + self.world.index_reward).long() * ( + t <= u + self.world.index_action ).long() - result[:, u - 1] = s + 1 + escape.first_lookahead_rewards_code - ar_mask = (t >= u).long() * (t < u + state_len).long() ar(result, ar_mask) - # Put the lookahead reward to +1 for the current - # iteration, sample the action and reward - s = 1 - result[:, u - 1] = s + 1 + escape.first_lookahead_rewards_code - ar_mask = (t >= u + state_len).long() * (t < u + state_len + 2).long() - ar(result, ar_mask) + # Set the lookahead_reward to UNKNOWN for the next iterations + result[ + :, u + self.world.index_lookahead_reward + ] = self.world.lookahead_reward2code(gree.REWARD_UNKNOWN) - # Fix the previous lookahead rewards in a consistant state - for v in range(0, u, it_len): - # Extract the rewards - r = result[:, range(v + state_len + 1 + it_len, u + it_len - 1, it_len)] - r = r - escape.first_rewards_code - 1 - r = r.clamp(min=-1, max=1) # the reward is predicted hence can be weird - a = r.min(dim=1).values - b = r.max(dim=1).values - s = (a < 0).long() * a + (a >= 0).long() * b - result[:, v + state_len + 2] = ( - s + 1 + escape.first_lookahead_rewards_code - ) + filename = os.path.join(result_dir, f"test_thinking_compute_{n_epoch:04d}.txt") + with open(filename, "w") as f: + for n in range(10): + for s in snapshots: + lr, s, a, r = self.world.seq2episodes( + s[n : n + 1], + ) + str = self.world.episodes2str( + lr, s, a, r, unicode=True, ansi_colors=True + ) + f.write(str) + f.write("\n\n") # Saving the generated sequences - s, a, r, lr = escape.seq2episodes( - result, self.height, self.width, lookahead=True - ) - str = escape.episodes2str( - s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True - ) + lr, s, a, r = self.world.seq2episodes(result) + str = self.world.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True) filename = os.path.join(result_dir, f"test_thinking_seq_{n_epoch:04d}.txt") with open(filename, "w") as f: @@ -1986,16 +2014,14 @@ class Escape(Task): def produce_results( self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 ): - result = self.test_input[:250].clone() + result = self.wipe_lookahead_rewards(self.test_input[:250].clone()) # Saving the ground truth - s, a, r, lr = escape.seq2episodes( - result, self.height, self.width, lookahead=True - ) - str = escape.episodes2str( - s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True + lr, s, a, r = self.world.seq2episodes( + result, ) + str = self.world.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True) filename = os.path.join(result_dir, f"test_true_seq_{n_epoch:04d}.txt") with open(filename, "w") as f: @@ -2005,8 +2031,7 @@ class Escape(Task): # Re-generating from the first frame ar_mask = ( - torch.arange(result.size(1), device=result.device) - >= self.height * self.width + 3 + torch.arange(result.size(1), device=result.device) >= self.world.it_len ).long()[None, :] ar_mask = ar_mask.expand_as(result) result *= 1 - ar_mask # paraaaaanoiaaaaaaa @@ -2022,12 +2047,10 @@ class Escape(Task): # Saving the generated sequences - s, a, r, lr = escape.seq2episodes( - result, self.height, self.width, lookahead=True - ) - str = escape.episodes2str( - s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True + lr, s, a, r = self.world.seq2episodes( + result, ) + str = self.world.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True) filename = os.path.join(result_dir, f"test_seq_{n_epoch:04d}.txt") with open(filename, "w") as f: