X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=quizz_machine.py;h=92b579980ec1a10aec128f8a91927f303417163c;hb=d3d4ce7bb2b799f4bf81a936987e3a8938514af8;hp=90f288ec1cc7597c067ed95787b2b1be5826a037;hpb=c3b4a2ec01b3ebe8e89664223f2b96ce5dc9a2ed;p=culture.git diff --git a/quizz_machine.py b/quizz_machine.py index 90f288e..92b5799 100755 --- a/quizz_machine.py +++ b/quizz_machine.py @@ -122,12 +122,13 @@ class QuizzMachine: forward_to_backward = torch.cat( [ quizzes[:, 0:1], - quizzes[:, 2 + self.prompt_len :], - quizzes[:, 1 + self.prompt_len : 2 + self.prompt_len], + quizzes[:, 2 + self.prompt_len : 2 + self.prompt_len + self.answer_len], + quizzes[:, 1 + self.prompt_len : 1 + self.prompt_len + 1], quizzes[:, 1 : 1 + self.prompt_len], ], dim=1, ) + forward_to_backward[:, 0] = self.token_backward forward_to_backward[:, 1 + self.answer_len] = self.token_backward @@ -202,6 +203,7 @@ class QuizzMachine: problem, nb_train_samples, nb_test_samples, + back_accuracy, batch_size, result_dir, logger, @@ -215,6 +217,7 @@ class QuizzMachine: self.nb_token_values = v + 2 self.problem = problem + self.back_accuracy = back_accuracy self.batch_size = batch_size self.device = device self.logger = logger @@ -232,26 +235,41 @@ class QuizzMachine: if result_dir is not None: self.save_quizzes( - result_dir, "culture_w_quizzes", self.train_w_quizzes[:72] + result_dir, + "culture_w_quizzes", + self.train_w_quizzes[:72], + n_backward=self.train_w_quizzes[:72, 0] == self.token_backward, ) - # toto = self.reverse_time(self.train_w_quizzes[:72]) - # self.save_quizzes(result_dir, "toto", toto) - # exit(0) - - def save_quizzes(self, result_dir, filename_prefix, quizzes, prediction=False): + def save_quizzes( + self, + result_dir, + filename_prefix, + quizzes, + n_backward=None, + mistakes=None, + ): + quizzes = quizzes.clone() forward = quizzes[quizzes[:, 0] == self.token_forward] ib = quizzes[:, 0] == self.token_backward backward = quizzes[ib] assert forward.size(0) + backward.size(0) == quizzes.size(0) quizzes[ib] = self.reverse_time(quizzes[ib]) - if prediction: - predicted_prompts = ib - predicted_answers = torch.logical_not(ib) - else: + if n_backward is None: predicted_prompts = None predicted_answers = None + else: + predicted_prompts = n_backward.long() + predicted_answers = 1 - predicted_prompts + if mistakes is not None: + # 0/-1/+1 ~ not-to-predict / predicted wrong / predicted correct + predicted_prompts *= mistakes + predicted_answers *= mistakes + else: + # 0/2 ~ not-to-predict / to predict + predicted_prompts *= 2 + predicted_answers *= 2 self.problem.save_quizzes( result_dir, @@ -307,8 +325,7 @@ class QuizzMachine: def produce_results( self, n_epoch, model, result_dir, deterministic_synthesis, nmax=1000 ): - def compute_accuracy(input): - input = input[:nmax] + def compute_accuracy(input, log_prefix=None): ar_mask = self.make_ar_mask(input) result = input.clone() * (1 - ar_mask) seq_logproba = torch.empty(input.size(0), device=self.device) @@ -325,50 +342,56 @@ class QuizzMachine: device=self.device, ) - nb_total = input.size(0) - nb_correct = (input == result).long().min(dim=1).values.sum() + correct = torch.empty(input.size(0), dtype=torch.int64, device=input.device) - return nb_total, nb_correct + n_forward = input[:, 0] == self.token_forward + n_backward = input[:, 0] == self.token_backward - train_nb_total, train_nb_correct = compute_accuracy(self.train_w_quizzes) + correct[n_forward] = ( + (input[n_forward] == result[n_forward]).long().min(dim=1).values + ) - self.logger( - f"accuracy_train {n_epoch} nb_total {train_nb_total} nb_correct {train_nb_correct} accuracy {(100.0*train_nb_correct)/train_nb_total:.02f}%" - ) + if self.back_accuracy and n_backward.any(): + # accuracy of B->A*->B*=B instead of B->A*=A + back_input = self.reverse_time(result[n_backward]) + back_input[:, 2 + self.prompt_len :] = input[ + n_backward, 1 : 1 + self.answer_len + ] + result[n_backward], correct[n_backward] = compute_accuracy(back_input) - test_nb_total, test_nb_correct = compute_accuracy(self.test_w_quizzes) + if log_prefix is not None: + forward_nb_correct = correct[n_forward].sum() + forward_nb_total = correct[n_forward].size(0) + backward_nb_correct = correct[n_backward].sum() + backward_nb_total = correct[n_backward].size(0) - self.logger( - f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%" - ) + self.logger( + f"forward_accuracy {log_prefix} {n_epoch} {model.id=} {forward_nb_correct} / {forward_nb_total}" + ) - main_test_accuracy = test_nb_correct / test_nb_total - self.logger(f"main_test_accuracy {n_epoch} {main_test_accuracy}") + self.logger( + f"backward_accuracy {log_prefix} {n_epoch} {model.id=} {backward_nb_correct} / {backward_nb_total}" + ) - ############################## + return result, correct - input = self.test_w_quizzes[:96] - ar_mask = self.make_ar_mask(input) - result = input.clone() * (1 - ar_mask) - seq_logproba = torch.empty(input.size(0), device=self.device) + compute_accuracy(self.train_w_quizzes[:nmax], log_prefix="train") - masked_inplace_autoregression( - model=model, - batch_size=self.batch_size, - input=result, - ar_mask=ar_mask, - seq_logproba=seq_logproba, - temperature=1.0, - deterministic_synthesis=deterministic_synthesis, - progress_bar_desc=None, - device=self.device, + test_result, test_correct = compute_accuracy( + self.test_w_quizzes[:nmax], log_prefix="test" ) + main_test_accuracy = test_correct.sum() / test_correct.size(0) + self.logger(f"main_test_accuracy {n_epoch} {main_test_accuracy}") + + ############################## + self.save_quizzes( result_dir, f"culture_prediction_{n_epoch:04d}_{model.id:02d}", - quizzes=result[:72], - prediction=True, + quizzes=test_result[:72], + n_backward=self.test_w_quizzes[:72, 0] == self.token_backward, + mistakes=test_correct[:72] * 2 - 1, ) return main_test_accuracy