X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;ds=sidebyside;f=quizz_machine.py;h=6cad6a1c74a5b3fc04d04ce253c65f94286cf06f;hb=db8c21397d370ae16fd6078858c649e2ab14fe4e;hp=239dc687cbb33264e330f305cabab73b28fbebbf;hpb=bfcef9a8c82ed45528601e85725166241bbee916;p=culture.git diff --git a/quizz_machine.py b/quizz_machine.py index 239dc68..6cad6a1 100755 --- a/quizz_machine.py +++ b/quizz_machine.py @@ -161,8 +161,8 @@ class QuizzMachine: nb_train_samples, nb_test_samples, batch_size, - result_dir=None, - logger=None, + result_dir, + logger, device=torch.device("cpu"), ): super().__init__() @@ -170,6 +170,7 @@ class QuizzMachine: self.problem = problem self.batch_size = batch_size self.device = device + self.logger = logger self.train_w_quizzes = self.problem.generate_token_sequences( nb_train_samples @@ -231,9 +232,9 @@ class QuizzMachine: return self.nb_codes def produce_results( - self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000 + self, n_epoch, model, result_dir, deterministic_synthesis, nmax=1000 ): - def compute_accuracy(input, logger=None): + def compute_accuracy(input): input = input[:nmax] ar_mask = self.make_ar_mask(input) result = input.clone() * (1 - ar_mask) @@ -260,18 +261,18 @@ class QuizzMachine: train_nb_total, train_nb_correct = compute_accuracy(self.train_w_quizzes) - logger( + self.logger( f"accuracy_train {n_epoch} nb_total {train_nb_total} nb_correct {train_nb_correct} accuracy {(100.0*train_nb_correct)/train_nb_total:.02f}%" ) - test_nb_total, test_nb_correct = compute_accuracy(self.test_w_quizzes, logger) + test_nb_total, test_nb_correct = compute_accuracy(self.test_w_quizzes) - logger( + self.logger( f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" ) main_test_accuracy = test_nb_correct / test_nb_total - logger(f"main_test_accuracy {n_epoch} {main_test_accuracy}") + self.logger(f"main_test_accuracy {n_epoch} {main_test_accuracy}") ############################## @@ -311,7 +312,6 @@ class QuizzMachine: self.test_c_quizzes.append(new_c_quizzes) def comput_correctness(self, c_quizzes, models_for_validation): - ############################################################### # Create the reverse quizzes token_forward, token_backward = self.problem.direction_tokens() @@ -328,11 +328,9 @@ class QuizzMachine: ar_mask = self.make_ar_mask(c_quizzes) seq_logproba = torch.empty(ar_mask.size(0), device=self.device) - ############################################################### - # Check how many of the other models can solve them in both - # directions + # Check how many of models can solve the quizzes in both directions - nb_correct = [] + nb_correct = 0 for model in models_for_validation: result = c_quizzes.clone() @@ -369,24 +367,26 @@ class QuizzMachine: (reverse_c_quizzes == reverse_result).long().min(dim=-1).values ) - nb_correct.append((correct * reverse_correct)[None, :]) + nb_correct += correct * reverse_correct - return torch.cat(nb_correct, dim=0).sum(dim=0) + return nb_correct - def generate_quizzes(self, nb, model_for_generation, min_ave_seq_logproba): - ############################################################### - # Generate quizzes with model + ############################################################### + def generate_quizzes(self, nb, model_for_generation, min_ave_seq_logproba): c_quizzes = torch.empty( nb, self.train_w_quizzes.size(1), device=self.device, dtype=torch.int64 ) - ar_mask = torch.full(c_quizzes.size(), 1, device=self.device) - seq_logproba = torch.empty(ar_mask.size(0), device=self.device) + ar_mask_prompt = torch.zeros(c_quizzes.size(), device=self.device) + ar_mask_prompt[:, : ar_mask_prompt.size(1) // 2 + 1] = 1 + ar_mask_solve = 1 - ar_mask_prompt + seq_logproba = torch.empty(ar_mask_prompt.size(0), device=self.device) - # bracketing of the temperature to get the target logproba + # bracketing of the temperature to get the target logproba if + # min_ave_seq_logproba is not None - temperature = 1 + temperature = 2 d_temperature = 1 / 3 while True: @@ -396,7 +396,7 @@ class QuizzMachine: model=model_for_generation, batch_size=self.batch_size, input=c_quizzes, - ar_mask=ar_mask, + ar_mask=ar_mask_prompt, seq_logproba=seq_logproba, temperature=temperature, deterministic_synthesis=False, @@ -406,6 +406,18 @@ class QuizzMachine: ave_seq_logproba = seq_logproba.mean() + masked_inplace_autoregression( + model=model_for_generation, + batch_size=self.batch_size, + input=c_quizzes, + ar_mask=ar_mask_solve, + seq_logproba=seq_logproba, + temperature=temperature, + deterministic_synthesis=True, + # progress_bar_desc="sampling c_quizzes", + device=self.device, + ) + # If we do not have target logprobs, get out now if min_ave_seq_logproba is None: break @@ -422,7 +434,7 @@ class QuizzMachine: else: break - logger(f"changing temperature to {temperature}") + self.logger(f"changing temperature to {temperature}") return c_quizzes, seq_logproba.mean() @@ -436,7 +448,6 @@ class QuizzMachine: min_ave_seq_logproba, n_epoch, result_dir, - logger, ): c_quizzes, ave_seq_logproba = self.generate_quizzes( nb, model_for_generation, min_ave_seq_logproba @@ -457,7 +468,6 @@ class QuizzMachine: min_ave_seq_logproba, n_epoch, result_dir, - logger, ): model_for_generation = Gang(models, nb_models_for_generation, mode) models_for_validation = models @@ -468,5 +478,4 @@ class QuizzMachine: min_ave_seq_logproba, n_epoch, result_dir, - logger, )