)
parser.add_argument(
- "--task", type=str, default="picoclvr", help="picoclvr, mnist, maze, snake, stack"
+ "--task",
+ type=str,
+ default="picoclvr",
+ help="picoclvr, mnist, maze, snake, stack, expr",
)
parser.add_argument("--log_filename", type=str, default="train.log", help=" ")
parser.add_argument("--stack_fraction_values_for_train", type=float, default=None)
+##############################
+# Expr options
+
+parser.add_argument("--expr_nb_variables", type=int, default=5)
+
+parser.add_argument("--expr_sequence_length", type=int, default=30)
+
######################################################################
args = parser.parse_args()
"nb_train_samples": 100000,
"nb_test_samples": 1000,
},
+ "expr": {
+ "nb_epochs": 5,
+ "batch_size": 25,
+ "nb_train_samples": 100000,
+ "nb_test_samples": 1000,
+ },
}
if args.task in default_args:
progress_bar_desc="autoregression",
device=torch.device("cpu"),
):
- # p = logits.softmax(1)
- # entropy[:,s]= p.xlogy(p).sum(1) / math.log(2)
batches = zip(input.split(batch_size), ar_mask.split(batch_size))
+
if progress_bar_desc is not None:
batches = tqdm.tqdm(
batches,
desc=progress_bar_desc,
total=input.size(0) // batch_size,
)
+
for input, ar_mask in batches:
i = (ar_mask.sum(0) > 0).nonzero()
if i.min() > 0:
######################################################################
+import expr
+
+
+class TaskExpr(Task):
+ def __init__(
+ self,
+ nb_train_samples,
+ nb_test_samples,
+ nb_variables,
+ sequence_length,
+ batch_size,
+ device=torch.device("cpu"),
+ ):
+ self.batch_size = batch_size
+ self.device = device
+
+ train_sequences = expr.generate_sequences(
+ nb_train_samples, nb_variables=nb_variables, length=sequence_length
+ )
+ test_sequences = expr.generate_sequences(
+ nb_test_samples, nb_variables=nb_variables, length=sequence_length
+ )
+ self.char2id = dict(
+ [
+ (c, n)
+ for n, c in enumerate(
+ set("#" + "".join(train_sequences + test_sequences))
+ )
+ ]
+ )
+ self.id2char = dict([(n, c) for c, n in self.char2id.items()])
+ len_max = max([len(x) for x in train_sequences + test_sequences])
+ self.train_input = torch.cat(
+ [
+ torch.tensor(
+ [
+ [self.char2id[c] for c in s + "#" * (len_max - len(s))]
+ for s in train_sequences
+ ]
+ )
+ ],
+ 0,
+ ).to(device)
+ self.test_input = torch.cat(
+ [
+ torch.tensor(
+ [
+ [self.char2id[c] for c in s + "#" * (len_max - len(s))]
+ for s in test_sequences
+ ]
+ )
+ ],
+ 0,
+ ).to(device)
+ self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1
+
+ def batches(self, split="train", nb_to_use=-1, desc=None):
+ assert split in {"train", "test"}
+ input = self.train_input if split == "train" else self.test_input
+ if nb_to_use > 0:
+ input = input[:nb_to_use]
+ if desc is None:
+ desc = f"epoch-{split}"
+ for batch in tqdm.tqdm(
+ input.split(self.batch_size), dynamic_ncols=True, desc=desc
+ ):
+ yield batch
+
+ def vocabulary_size(self):
+ return self.nb_codes
+
+ def produce_results(self, n_epoch, model):
+ with torch.autograd.no_grad():
+ t = model.training
+ model.eval()
+
+ def compute_nb_correct(input):
+ result = input.clone()
+ filler, space = self.char2id["#"], self.char2id[" "]
+ ar_mask = (result == space).long().cumsum(dim=1).clamp(max=1)
+ result = (1 - ar_mask) * result + filler * ar_mask
+ masked_inplace_autoregression(
+ model, self.batch_size, result, ar_mask, device=self.device
+ )
+
+ nb_total = ar_mask.sum()
+ nb_correct = ((input == result).long() * ar_mask).sum()
+
+ return nb_total, nb_correct
+
+ test_nb_total, test_nb_correct = compute_nb_correct(self.test_input[:1000])
+
+ log_string(
+ f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%"
+ )
+
+ ##############################################################
+ # Log a few generated sequences
+ input = self.test_input[:10]
+ result = input.clone()
+ filler, space = self.char2id["#"], self.char2id[" "]
+ ar_mask = (result == space).long().cumsum(dim=1).clamp(max=1)
+ result = (1 - ar_mask) * result + filler * ar_mask
+ for n in range(result.size(0)):
+ s = "".join([self.id2char[k.item()] for k in result[n]])
+ log_string(f"test_before {s}")
+ masked_inplace_autoregression(
+ model, self.batch_size, result, ar_mask, device=self.device
+ )
+ for n in range(result.size(0)):
+ s = "".join([self.id2char[k.item()] for k in result[n]])
+ log_string(f"test_after {s}")
+ ##############################################################
+
+ model.train(t)
+
+
+######################################################################
+
+
def picoclvr_pruner_horizontal_green(p):
return not ("green" in p and ("left" in p or "right" in p))
device=device,
)
+elif args.task == "expr":
+ task = TaskExpr(
+ nb_train_samples=args.nb_train_samples,
+ nb_test_samples=args.nb_test_samples,
+ nb_variables=args.expr_nb_variables,
+ sequence_length=args.expr_sequence_length,
+ batch_size=args.batch_size,
+ device=device,
+ )
+
else:
raise ValueError(f"Unknown task {args.task}")