X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=tasks.py;h=183c3cfc0ff7faeae97a4ec2de2dd529ded3192b;hb=0cba1df2952a9f9b88b6e7aacfcddc17fbc35186;hp=cc3aea0287574076abc4dc220f338494ad9d4f97;hpb=59600257e0eda86816a43676c5ffbe598d78bdb5;p=picoclvr.git diff --git a/tasks.py b/tasks.py index cc3aea0..183c3cf 100755 --- a/tasks.py +++ b/tasks.py @@ -181,6 +181,8 @@ class SandBox(Task): f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) + logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}") + if save_attention_image is None: logger("no save_attention_image (is pycairo installed?)") else: @@ -371,6 +373,10 @@ class PicoCLVR(Task): f"property_{prefix}miss {n_epoch} {100*nb_missing_properties/nb_requested_properties:.02f}%" ) + logger( + f"main_test_accuracy {n_epoch} {1-nb_missing_properties/nb_requested_properties}" + ) + ###################################################################### def produce_results( @@ -641,6 +647,8 @@ class Maze(Task): f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) + logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}") + if count is not None: proportion_optimal = count.diagonal().sum().float() / count.sum() logger(f"proportion_optimal_test {proportion_optimal*100:.02f}%") @@ -780,6 +788,8 @@ class Snake(Task): f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) + logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}") + ###################################################################### @@ -889,6 +899,8 @@ class Stack(Task): f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) + logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}") + ############################################################## # Log a few generated sequences input = self.test_input[:10, : 12 * (1 + self.nb_digits)] @@ -1161,6 +1173,8 @@ class RPL(Task): f"accuracy_prog_test {n_epoch} nb_total {test_nb_total} nb_errors {test_nb_errors} accuracy {100.0*(1-test_nb_errors/test_nb_total):.02f}%" ) + logger(f"main_test_accuracy {n_epoch} {1-test_nb_errors/test_nb_total}") + test_nb_total, test_nb_errors = compute_nb_errors_output( self.test_input[:1000].to(self.device), nb_to_log=10 ) @@ -1359,6 +1373,8 @@ class Expr(Task): f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) + logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}") + nb_total = test_nb_delta.sum() + test_nb_missed for d in range(test_nb_delta.size(0)): logger( @@ -1405,95 +1421,110 @@ class Expr(Task): ###################################################################### -import world +import grid + + +class Grid(Task): + # Make a tensor from a list of strings + def str2tensor(self, descr): + token_descr = [s.strip().split(" ") for s in descr] + l = max([len(s) for s in token_descr]) + token_descr = [s + ["#"] * (l - len(s)) for s in token_descr] + id_descr = [[self.token2id[u] for u in s] for s in token_descr] + return torch.tensor(id_descr, device=self.device) + + # Make a list of strings from a tensor + def tensor2str(self, x): + return [" ".join([self.id2token[t.item()] for t in r]) for r in x] + + # trim all the tensors in the tuple z to remove as much token from + # left and right in the first tensor. If z is a tuple, all its + # elements are trimed according to the triming for the first + def trim(self, z, token="#"): + n = self.token2id[token] + if type(z) == tuple: + x = z[0] + i = (1 - (F.pad(x, (1, 1), value=n) == n).min(0).values.long()).cumsum(0) + a, b = (i == 0).nonzero().max(), (i == i.max()).nonzero().min() + return tuple([t[:, a:b] for t in z]) + else: + i = (1 - (F.pad(z, (1, 1), value=n) == n).min(0).values.long()).cumsum(0) + a, b = (i == 0).nonzero().max(), (i == i.max()).nonzero().min() + return z[:, a:b] + ###################### -class World(Task): def __init__( self, nb_train_samples, nb_test_samples, batch_size, - vqae_nb_epochs, + size, logger=None, device=torch.device("cpu"), - device_storage=torch.device("cpu"), ): super().__init__() - self.batch_size = batch_size self.device = device + self.batch_size = batch_size + self.grid_factory = grid.GridFactory(size=size) - ( - train_frames, - train_action_seq, - test_frames, - test_action_seq, - self.frame2seq, - self.seq2frame, - ) = world.create_data_and_processors( - nb_train_samples, - nb_test_samples, - mode="first_last", - nb_steps=30, - nb_epochs=vqae_nb_epochs, - logger=logger, - device=device, - device_storage=device_storage, - ) - - train_frame_seq = self.frame2seq(train_frames).to(device_storage) - test_frame_seq = self.frame2seq(test_frames).to(device_storage) - - nb_frame_codes = max(train_frame_seq.max(), test_frame_seq.max()) + 1 - nb_action_codes = max(train_action_seq.max(), test_action_seq.max()) + 1 - - self.len_frame_seq = train_frame_seq.size(1) - self.len_action_seq = train_action_seq.size(1) - self.nb_codes = nb_frame_codes + nb_action_codes - - train_frame_seq = train_frame_seq.reshape(train_frame_seq.size(0) // 2, 2, -1) + if logger is not None: + logger( + f"generating {nb_train_samples+nb_test_samples} samples (can take some time)" + ) - train_action_seq += nb_frame_codes - self.train_input = torch.cat( - (train_frame_seq[:, 0, :], train_action_seq, train_frame_seq[:, 1, :]), 1 + self.train_descr = self.grid_factory.generate_samples( + nb_train_samples, lambda r: tqdm.tqdm(r) ) - - test_frame_seq = test_frame_seq.reshape(test_frame_seq.size(0) // 2, 2, -1) - test_action_seq += nb_frame_codes - self.test_input = torch.cat( - (test_frame_seq[:, 0, :], test_action_seq, test_frame_seq[:, 1, :]), 1 + self.test_descr = self.grid_factory.generate_samples( + nb_test_samples, lambda r: tqdm.tqdm(r) ) - def batches(self, split="train", nb_to_use=-1, desc=None): + # Build the tokenizer + tokens = set() + for d in [self.train_descr, self.test_descr]: + for s in d: + for t in s.strip().split(" "): + tokens.add(t) + # make this set a sorted list to get the same tensors given + # the same descr + tokens = list(tokens) + tokens.sort() + tokens = ["#"] + tokens + self.token2id = dict([(t, n) for n, t in enumerate(tokens)]) + self.id2token = dict([(n, t) for n, t in enumerate(tokens)]) + self.t_nul = self.token2id["#"] + self.t_true = self.token2id["true"] + self.t_false = self.token2id["false"] + + # Tokenize the train and test sets + self.train_input = self.str2tensor(self.train_descr) + self.test_input = self.str2tensor(self.test_descr) + + def batches(self, split="train"): assert split in {"train", "test"} input = self.train_input if split == "train" else self.test_input - if nb_to_use > 0: - input = input[:nb_to_use] - if desc is None: - desc = f"epoch-{split}" for batch in tqdm.tqdm( - input.split(self.batch_size), dynamic_ncols=True, desc=desc + input.split(self.batch_size), dynamic_ncols=True, desc=f"epoch-{split}" ): - yield batch.to(self.device) + yield self.trim(batch) def vocabulary_size(self): - return self.nb_codes + return len(self.token2id) def produce_results( self, n_epoch, model, result_dir, logger, deterministic_synthesis ): - k = torch.arange( - 2 * self.len_frame_seq + self.len_action_seq, device=self.device - )[None, :] + correct = self.test_input[:1000] + result = correct.clone() + ar_mask = torch.logical_or(result == self.t_true, result == self.t_false).long() + result *= 1 - ar_mask # paraaaaanoiaaaaaaa - input = self.test_input[:64].to(self.device) - result = input.clone() + logger(f"----------------------------------------------------------") - ar_mask = ( - (k >= self.len_frame_seq + self.len_action_seq).long().expand_as(result) - ) - result *= 1 - ar_mask + for e in self.tensor2str(result[:10]): + logger(f"test_before {e}") masked_inplace_autoregression( model, @@ -1504,25 +1535,18 @@ class World(Task): device=self.device, ) - seq_start = input[:, : self.len_frame_seq] - seq_end = input[:, self.len_frame_seq + self.len_action_seq :] - seq_predicted = result[:, self.len_frame_seq + self.len_action_seq :] + logger(f"----------------------------------------------------------") - result = torch.cat( - (seq_start[:, None, :], seq_end[:, None, :], seq_predicted[:, None, :]), 1 - ) - result = result.reshape(-1, result.size(-1)) + for e in self.tensor2str(result[:10]): + logger(f"test_after {e}") - frames = self.seq2frame(result) - image_name = os.path.join(result_dir, f"world_result_{n_epoch:04d}.png") - torchvision.utils.save_image( - frames.float() / (world.Box.nb_rgb_levels - 1), - image_name, - nrow=12, - padding=1, - pad_value=0.0, - ) - logger(f"wrote {image_name}") + logger(f"----------------------------------------------------------") + + nb_total = ar_mask.sum().item() + nb_correct = ((correct == result).long() * ar_mask).sum().item() + + logger(f"test_performance {n_epoch} {nb_total=} {nb_correct=}") + logger(f"main_test_accuracy {n_epoch} {nb_correct / nb_total}") ######################################################################