X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=main.py;h=9dee679fbf1bdcda0faac54cc77179072c4ad0a4;hb=ca5b98d1517b8ce2367887bbad2205f27d55e0b3;hp=14b1bc346064733faa1022e80c5278005a1bc359;hpb=c5daf2eeedb26a25789de370171d592c621a2fac;p=picoclvr.git diff --git a/main.py b/main.py index 14b1bc3..9dee679 100755 --- a/main.py +++ b/main.py @@ -32,12 +32,15 @@ parser = argparse.ArgumentParser( ) parser.add_argument( - "--task", type=str, default="picoclvr", help="picoclvr, mnist, maze, snake, stack" + "--task", + type=str, + default="picoclvr", + help="picoclvr, mnist, maze, snake, stack, expr", ) parser.add_argument("--log_filename", type=str, default="train.log", help=" ") -parser.add_argument("--result_dir", type=str, default="results_default") +parser.add_argument("--result_dir", type=str, default=None) parser.add_argument("--seed", type=int, default=0) @@ -45,9 +48,9 @@ parser.add_argument("--nb_epochs", type=int, default=None) parser.add_argument("--batch_size", type=int, default=None) -parser.add_argument("--nb_train_samples", type=int, default=250000) +parser.add_argument("--nb_train_samples", type=int, default=None) -parser.add_argument("--nb_test_samples", type=int, default=10000) +parser.add_argument("--nb_test_samples", type=int, default=None) parser.add_argument("--optim", type=str, default="adam") @@ -113,7 +116,16 @@ parser.add_argument("--stack_nb_steps", type=int, default=100) parser.add_argument("--stack_nb_stacks", type=int, default=1) -parser.add_argument("--stack_nb_values", type=int, default=10) +parser.add_argument("--stack_nb_digits", type=int, default=3) + +parser.add_argument("--stack_fraction_values_for_train", type=float, default=None) + +############################## +# Expr options + +parser.add_argument("--expr_nb_variables", type=int, default=5) + +parser.add_argument("--expr_sequence_length", type=int, default=30) ###################################################################### @@ -121,22 +133,8 @@ args = parser.parse_args() assert args.picocvlr_prune_properties in {"none", "train+eval", "eval"} -try: - os.mkdir(args.result_dir) -except FileExistsError: - if not args.overwrite_results: - print(f"result directory {args.result_dir} already exists") - exit(1) - -log_file = open(os.path.join(args.result_dir, args.log_filename), "a") - -if args.seed >= 0: - # torch.backends.cudnn.deterministic = True - # torch.backends.cudnn.benchmark = False - # torch.use_deterministic_algorithms(True) - torch.manual_seed(args.seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(args.seed) +if args.result_dir is None: + args.result_dir = f"results_{args.task}" ###################################################################### @@ -171,6 +169,12 @@ default_args = { "nb_train_samples": 100000, "nb_test_samples": 1000, }, + "expr": { + "nb_epochs": 50, + "batch_size": 25, + "nb_train_samples": 250000, + "nb_test_samples": 10000, + }, } if args.task in default_args: @@ -180,6 +184,25 @@ if args.task in default_args: ###################################################################### +try: + os.mkdir(args.result_dir) +except FileExistsError: + if not args.overwrite_results: + print(f"result directory {args.result_dir} already exists") + exit(1) + +log_file = open(os.path.join(args.result_dir, args.log_filename), "a") + +if args.seed >= 0: + # torch.backends.cudnn.deterministic = True + # torch.backends.cudnn.benchmark = False + # torch.use_deterministic_algorithms(True) + torch.manual_seed(args.seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(args.seed) + +###################################################################### + def log_string(s): t = time.strftime("%Y%m%d-%H:%M:%S ", time.localtime()) @@ -210,16 +233,16 @@ def masked_inplace_autoregression( progress_bar_desc="autoregression", device=torch.device("cpu"), ): - # p = logits.softmax(1) - # entropy[:,s]= p.xlogy(p).sum(1) / math.log(2) batches = zip(input.split(batch_size), ar_mask.split(batch_size)) + if progress_bar_desc is not None: - tqdm.tqdm( + batches = tqdm.tqdm( batches, dynamic_ncols=True, desc=progress_bar_desc, total=input.size(0) // batch_size, ) + for input, ar_mask in batches: i = (ar_mask.sum(0) > 0).nonzero() if i.min() > 0: @@ -702,14 +725,14 @@ class TaskMaze(Task): model, "train", nb_to_use=1000 ) log_string( - f"accuracy_train nb_total {train_nb_total} nb_correct {train_nb_correct} accuracy {(100.0*train_nb_correct)/train_nb_total:.02f}%" + f"accuracy_train {n_epoch} nb_total {train_nb_total} nb_correct {train_nb_correct} accuracy {(100.0*train_nb_correct)/train_nb_total:.02f}%" ) test_nb_total, test_nb_correct, count = self.compute_error( model, "test", nb_to_use=1000 ) log_string( - f"accuracy_test nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" + f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) if count is not None: @@ -855,7 +878,7 @@ class TaskSnake(Task): ) log_string( - f"accuracy_test nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" + f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) model.train(t) @@ -875,29 +898,47 @@ class TaskStack(Task): batch_size, nb_steps, nb_stacks, - nb_values, + nb_digits, + fraction_values_for_train=None, device=torch.device("cpu"), ): self.batch_size = batch_size self.nb_steps = nb_steps self.nb_stacks = nb_stacks - self.nb_values = nb_values + self.nb_digits = nb_digits self.device = device + if fraction_values_for_train is None: + values_for_train = None + values_for_test = None + else: + all = torch.randperm(10**nb_digits) + nb_for_train = int(all.size(0) * fraction_values_for_train) + values_for_train = all[:nb_for_train] + values_for_test = all[nb_for_train:] + self.train_input, self.train_stack_counts = stack.generate_sequences( - nb_train_samples, nb_steps, nb_stacks, nb_values, self.device + nb_train_samples, + nb_steps, + nb_stacks, + nb_digits, + values_for_train, + self.device, ) self.test_input, self.test_stack_counts = stack.generate_sequences( - nb_test_samples, nb_steps, nb_stacks, nb_values, self.device + nb_test_samples, + nb_steps, + nb_stacks, + nb_digits, + values_for_test, + self.device, ) - mask = self.test_input.clone() - stack.remove_poped_values(mask,self.nb_stacks) - mask=(mask!=self.test_input) - counts = self.test_stack_counts.flatten()[mask.flatten()] - counts=F.one_hot(counts).sum(0) - log_string(f"stack_count {counts}") + i = torch.logical_and(self.test_input % 2 == 1, self.test_input < 2 * nb_stacks) + counts = self.test_stack_counts.flatten()[i.flatten()] + counts = F.one_hot(counts).sum(0) + log_string(f"test_pop_stack_counts {counts}") self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1 @@ -923,28 +964,221 @@ class TaskStack(Task): def compute_nb_correct(input): result = input.clone() - stack.remove_poped_values(result,self.nb_stacks) + stack.remove_popped_values(result, self.nb_stacks, self.nb_digits) ar_mask = (result != input).long() - result *= 1 - ar_mask - masked_inplace_autoregression( model, self.batch_size, result, ar_mask, device=self.device ) - nb_total = ar_mask.sum() + errors = ((result != input).long() * ar_mask).reshape( + -1, 1 + self.nb_digits + ) + ar_mask = ar_mask.reshape(-1, 1 + self.nb_digits) - nb_correct = ( - (result == input).long() * ar_mask - ).sum() + nb_total = ar_mask.max(1).values.sum() + nb_correct = nb_total - errors.max(1).values.sum() return nb_total, nb_correct test_nb_total, test_nb_correct = compute_nb_correct(self.test_input[:1000]) log_string( - f"accuracy_test nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" + f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) + ############################################################## + # Log a few generated sequences + input = self.test_input[:10, : 12 * (1 + self.nb_digits)] + result = input.clone() + stack.remove_popped_values(result, self.nb_stacks, self.nb_digits) + ar_mask = (result != input).long() + for n in range(result.size(0)): + log_string( + f"test_before {stack.seq_to_str(result[n],nb_stacks=self.nb_stacks,nb_digits=self.nb_digits)}" + ) + masked_inplace_autoregression( + model, self.batch_size, result, ar_mask, device=self.device + ) + for n in range(result.size(0)): + log_string( + f"test_after {stack.seq_to_str(result[n],nb_stacks=self.nb_stacks,nb_digits=self.nb_digits)}" + ) + ############################################################## + + model.train(t) + + +###################################################################### + + +import expr + + +class TaskExpr(Task): + def __init__( + self, + nb_train_samples, + nb_test_samples, + nb_variables, + sequence_length, + batch_size, + device=torch.device("cpu"), + ): + self.batch_size = batch_size + self.device = device + + train_sequences = expr.generate_sequences( + nb_train_samples, + nb_variables=nb_variables, + length=sequence_length, + # length=2 * sequence_length, + # randomize_length=True, + ) + test_sequences = expr.generate_sequences( + nb_test_samples, + nb_variables=nb_variables, + length=sequence_length, + ) + self.char2id = dict( + [ + (c, n) + for n, c in enumerate( + set("#" + "".join(train_sequences + test_sequences)) + ) + ] + ) + self.id2char = dict([(n, c) for c, n in self.char2id.items()]) + + self.filler, self.space = self.char2id["#"], self.char2id[" "] + + len_max = max([len(x) for x in train_sequences]) + self.train_input = torch.cat( + [ + torch.tensor( + [ + [self.char2id[c] for c in s + "#" * (len_max - len(s))] + for s in train_sequences + ] + ) + ], + 0, + ).to(device) + + len_max = max([len(x) for x in test_sequences]) + self.test_input = torch.cat( + [ + torch.tensor( + [ + [self.char2id[c] for c in s + "#" * (len_max - len(s))] + for s in test_sequences + ] + ) + ], + 0, + ).to(device) + + self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1 + + def batches(self, split="train", nb_to_use=-1, desc=None): + assert split in {"train", "test"} + input = self.train_input if split == "train" else self.test_input + if nb_to_use > 0: + input = input[:nb_to_use] + if desc is None: + desc = f"epoch-{split}" + for batch in tqdm.tqdm( + input.split(self.batch_size), dynamic_ncols=True, desc=desc + ): + if split == "train": + last = (batch != self.filler).max(0).values.nonzero().max() + 1 + batch = batch[:, :last] + yield batch + + def vocabulary_size(self): + return self.nb_codes + + def seq2str(self, s): + return "".join([self.id2char[k.item()] for k in s]) + + def produce_results(self, n_epoch, model): + with torch.autograd.no_grad(): + t = model.training + model.eval() + + def compute_nb_correct(input): + result = input.clone() + ar_mask = (result == self.space).long().cumsum(dim=1).clamp(max=1) + result = (1 - ar_mask) * result + ar_mask * self.filler + masked_inplace_autoregression( + model, self.batch_size, result, ar_mask, device=self.device + ) + + nb_total = input.size(0) + nb_correct = (input == result).long().min(1).values.sum() + + ####################################################################### + # Comput predicted vs. true variable values + + nb_delta = torch.zeros(5, dtype=torch.int64) + nb_missed = 0 + + values_input = expr.extract_results([self.seq2str(s) for s in input]) + values_result = expr.extract_results([self.seq2str(s) for s in result]) + + for i, r in zip(values_input, values_result): + for n, vi in i.items(): + vr = r.get(n) + if vr is None or vr < 0: + nb_missed += 1 + else: + d = abs(vr - vi) + if d >= nb_delta.size(0): + nb_missed += 1 + else: + nb_delta[d] += 1 + + ###################################################################### + + return nb_total, nb_correct, nb_delta, nb_missed + + ( + test_nb_total, + test_nb_correct, + test_nb_delta, + test_nb_missed, + ) = compute_nb_correct(self.test_input[:1000]) + + log_string( + f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" + ) + + nb_total = test_nb_delta.sum() + test_nb_missed + for d in range(test_nb_delta.size(0)): + log_string( + f"error_value {n_epoch} delta {d} {test_nb_delta[d]} {test_nb_delta[d]*100/nb_total:.02f}%" + ) + log_string( + f"error_value {n_epoch} missed {test_nb_missed} {test_nb_missed*100/nb_total:.02f}%" + ) + + ############################################################## + # Log a few generated sequences + input = self.test_input[:10] + result = input.clone() + ar_mask = (result == self.space).long().cumsum(dim=1).clamp(max=1) + result = (1 - ar_mask) * result + ar_mask * self.filler + for n in range(result.size(0)): + log_string(f"test_before {self.seq2str(result[n])}") + masked_inplace_autoregression( + model, self.batch_size, result, ar_mask, device=self.device + ) + correct = (1 - ar_mask) * self.space + ar_mask * input + for n in range(result.size(0)): + comment = "GOOD" if (result[n] - input[n]).abs().max() == 0 else "" + log_string(f"test_after {self.seq2str(result[n])} {comment}") + log_string(f"correct {self.seq2str(correct[n])}") + ############################################################## + model.train(t) @@ -1017,9 +1251,20 @@ elif args.task == "stack": nb_train_samples=args.nb_train_samples, nb_test_samples=args.nb_test_samples, batch_size=args.batch_size, - nb_steps = args.stack_nb_steps, - nb_stacks = args.stack_nb_stacks, - nb_values = args.stack_nb_values, + nb_steps=args.stack_nb_steps, + nb_stacks=args.stack_nb_stacks, + nb_digits=args.stack_nb_digits, + fraction_values_for_train=args.stack_fraction_values_for_train, + device=device, + ) + +elif args.task == "expr": + task = TaskExpr( + nb_train_samples=args.nb_train_samples, + nb_test_samples=args.nb_test_samples, + nb_variables=args.expr_nb_variables, + sequence_length=args.expr_sequence_length, + batch_size=args.batch_size, device=device, )