From: François Fleuret Date: Fri, 21 Jun 2024 13:26:16 +0000 (+0200) Subject: Update. X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;ds=inline;h=b9924ef4309a69af2af150dba73125ed5fed0093;p=culture.git Update. --- diff --git a/main.py b/main.py index 18b19db..35f02a3 100755 --- a/main.py +++ b/main.py @@ -853,7 +853,7 @@ def one_epoch(model, task, learning_rate): train_perplexity = math.exp(min(100, acc_train_loss / nb_train_samples)) - log_string(f"train)perplexity {n_epoch} {train_perplexity}") + log_string(f"train_perplexity {n_epoch} {train_perplexity}") ###################################################################### @@ -887,7 +887,7 @@ def run_tests(model, task, deterministic_synthesis): ) test_perplexity = math.exp(min(100, acc_test_loss / nb_test_samples)) - log_string(f"test)perplexity {n_epoch} {test_perplexity}") + log_string(f"test_perplexity {n_epoch} {test_perplexity}") ###################################################################### @@ -897,7 +897,29 @@ for n_epoch in range(nb_epochs_finished, args.nb_epochs): one_epoch(model, task, learning_rate) - run_tests(model, task, deterministic_synthesis=True) + run_tests(model, task, deterministic_synthesis=False) + + # -------------------------------------------- + + if n_epoch >= 3: + nb_required = 1000 + kept = [] + + while sum([x.size(0) for x in kept]) < nb_required: + new_problems, nb_correct = task.create_new_problems( + n_epoch=n_epoch, + result_dir=args.result_dir, + logger=log_string, + nb=nb_required, + model=model, + nb_runs=10, + ) + + to_keep = new_problems[torch.logical_and(nb_correct >= 8, nb_correct < 10)] + log_string(f"keep {to_keep.size(0)} problems") + kept.append(to_keep) + + new_problems = torch.cat(kept, dim=0)[:nb_required] # -------------------------------------------- diff --git a/tasks.py b/tasks.py index b4e6f67..1b28108 100755 --- a/tasks.py +++ b/tasks.py @@ -395,133 +395,6 @@ class SandBox(Task): # logger(f"wrote {filename}") -###################################################################### - -import world - - -class World(Task): - def __init__( - self, - nb_train_samples, - nb_test_samples, - batch_size, - logger=None, - device=torch.device("cpu"), - ): - super().__init__() - - self.batch_size = batch_size - self.device = device - self.height = 6 - self.width = 8 - - self.train_input = world.generate( - nb_train_samples, height=self.height, width=self.width - ) - self.train_ar_mask = ( - (torch.arange(self.train_input.size(1)) > self.train_input.size(1) // 2) - .long()[None, :] - .expand_as(self.train_input) - ) - - self.test_input = world.generate( - nb_test_samples, height=self.height, width=self.width - ) - self.test_ar_mask = ( - (torch.arange(self.test_input.size(1)) > self.test_input.size(1) // 2) - .long()[None, :] - .expand_as(self.test_input) - ) - - self.train_input, self.train_ar_mask = self.train_input.to( - device - ), self.train_ar_mask.to(device) - self.test_input, self.test_ar_mask = self.test_input.to( - device - ), self.test_ar_mask.to(device) - - self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1 - - def batches(self, split="train", nb_to_use=-1, desc=None): - assert split in {"train", "test"} - input = self.train_input if split == "train" else self.test_input - if nb_to_use > 0: - input = input[:nb_to_use] - if desc is None: - desc = f"epoch-{split}" - for batch in tqdm.tqdm( - input.split(self.batch_size), dynamic_ncols=True, desc=desc - ): - yield batch - - def vocabulary_size(self): - return self.nb_codes - - def produce_results( - self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 - ): - def compute_accuracy(input, ar_mask, logger=None): - input, ar_mask = input[:nmax], ar_mask[:nmax] - result = input.clone() * (1 - ar_mask) - - masked_inplace_autoregression( - model, - self.batch_size, - result, - ar_mask, - deterministic_synthesis, - progress_bar_desc=None, - device=self.device, - ) - - nb_total, nb_correct = ( - input.size(0), - (input == result).long().min(dim=1).values.sum(), - ) - - return nb_total, nb_correct - - train_nb_total, train_nb_correct = compute_accuracy( - self.train_input, self.train_ar_mask - ) - - logger( - f"accuracy_train {n_epoch} nb_total {train_nb_total} nb_correct {train_nb_correct} accuracy {(100.0*train_nb_correct)/train_nb_total:.02f}%" - ) - - test_nb_total, test_nb_correct = compute_accuracy( - self.test_input, self.test_ar_mask, logger - ) - - logger( - f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" - ) - - logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}") - - ############################## - - input, ar_mask = self.test_input[:64], self.test_ar_mask[:64] - result = input.clone() * (1 - ar_mask) - - masked_inplace_autoregression( - model, - self.batch_size, - result, - ar_mask, - deterministic_synthesis, - progress_bar_desc=None, - device=self.device, - ) - - img = world.sample2img(result.to("cpu"), self.height, self.width) - - image_name = os.path.join(result_dir, f"world_result_{n_epoch:04d}.png") - torchvision.utils.save_image(img.float() / 255.0, image_name, nrow=8, padding=2) - logger(f"wrote {image_name}") - - ###################################################################### import picoclvr @@ -2220,3 +2093,176 @@ class Greed(Task): ###################################################################### +###################################################################### + +import world + + +class World(Task): + def __init__( + self, + nb_train_samples, + nb_test_samples, + batch_size, + logger=None, + device=torch.device("cpu"), + ): + super().__init__() + + self.batch_size = batch_size + self.device = device + self.height = 6 + self.width = 8 + + self.train_input = world.generate( + nb_train_samples, height=self.height, width=self.width + ) + self.train_ar_mask = ( + (torch.arange(self.train_input.size(1)) > self.train_input.size(1) // 2) + .long()[None, :] + .expand_as(self.train_input) + ) + + self.test_input = world.generate( + nb_test_samples, height=self.height, width=self.width + ) + self.test_ar_mask = ( + (torch.arange(self.test_input.size(1)) > self.test_input.size(1) // 2) + .long()[None, :] + .expand_as(self.test_input) + ) + + self.train_input, self.train_ar_mask = self.train_input.to( + device + ), self.train_ar_mask.to(device) + self.test_input, self.test_ar_mask = self.test_input.to( + device + ), self.test_ar_mask.to(device) + + self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1 + + def batches(self, split="train", nb_to_use=-1, desc=None): + assert split in {"train", "test"} + input = self.train_input if split == "train" else self.test_input + if nb_to_use > 0: + input = input[:nb_to_use] + if desc is None: + desc = f"epoch-{split}" + for batch in tqdm.tqdm( + input.split(self.batch_size), dynamic_ncols=True, desc=desc + ): + yield batch + + def vocabulary_size(self): + return self.nb_codes + + def produce_results( + self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 + ): + def compute_accuracy(input, ar_mask, logger=None): + input, ar_mask = input[:nmax], ar_mask[:nmax] + result = input.clone() * (1 - ar_mask) + + masked_inplace_autoregression( + model, + self.batch_size, + result, + ar_mask, + deterministic_synthesis, + progress_bar_desc=None, + device=self.device, + ) + + nb_total, nb_correct = ( + input.size(0), + (input == result).long().min(dim=1).values.sum(), + ) + + return nb_total, nb_correct + + train_nb_total, train_nb_correct = compute_accuracy( + self.train_input, self.train_ar_mask + ) + + logger( + f"accuracy_train {n_epoch} nb_total {train_nb_total} nb_correct {train_nb_correct} accuracy {(100.0*train_nb_correct)/train_nb_total:.02f}%" + ) + + test_nb_total, test_nb_correct = compute_accuracy( + self.test_input, self.test_ar_mask, logger + ) + + logger( + f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" + ) + + logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}") + + ############################## + + input, ar_mask = self.test_input[:64], self.test_ar_mask[:64] + result = input.clone() * (1 - ar_mask) + + masked_inplace_autoregression( + model, + self.batch_size, + result, + ar_mask, + deterministic_synthesis, + progress_bar_desc=None, + device=self.device, + ) + + img = world.sample2img(result.to("cpu"), self.height, self.width) + image_name = os.path.join(result_dir, f"world_result_{n_epoch:04d}.png") + torchvision.utils.save_image(img.float() / 255.0, image_name, nrow=8, padding=2) + logger(f"wrote {image_name}") + + def create_new_problems(self, n_epoch, result_dir, logger, nb, model, nb_runs): + new_problems = torch.empty( + nb, self.height * self.width * 2 + 1, device=self.device, dtype=torch.int64 + ) + ar_mask = torch.full(new_problems.size(), 1, device=self.device) + + masked_inplace_autoregression( + model, + self.batch_size, + new_problems, + ar_mask, + deterministic_synthesis=False, + progress_bar_desc="new problems", + device=self.device, + ) + + img = world.sample2img(new_problems[:64].to("cpu"), self.height, self.width) + image_name = os.path.join(result_dir, f"world_new_{n_epoch:04d}.png") + torchvision.utils.save_image(img.float() / 255.0, image_name, nrow=8, padding=2) + logger(f"wrote {image_name}") + + nb_correct = torch.empty(nb, device=self.device, dtype=torch.int64) + + for n in tqdm.tqdm( + range(new_problems.size(0)), dynamic_ncols=True, desc="checking problems" + ): + result = new_problems[n][None, :].expand(nb_runs, -1).clone() + ar_mask = ( + (torch.arange(result.size(1), device=self.device) > result.size(1) // 2) + .long()[None, :] + .expand_as(result) + ) + + masked_inplace_autoregression( + model, + self.batch_size, + result, + ar_mask, + deterministic_synthesis=False, + progress_bar_desc=None, + device=self.device, + ) + + nb_correct[n] = ( + (new_problems[n][None, :] == result).long().min(dim=1).values.sum() + ) + + return new_problems, nb_correct