X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=quizz_machine.py;h=697f27ece4e353fe8a264657013a7fe2f693a630;hb=c9c018e4c19ce92892d7652082fb90719d57441c;hp=eae256ba8fe0ee54b8dfcd1888df17f99526febf;hpb=66d210bd5e04ae58f9e1495df77f1f975ee99c56;p=culture.git diff --git a/quizz_machine.py b/quizz_machine.py index eae256b..697f27e 100755 --- a/quizz_machine.py +++ b/quizz_machine.py @@ -17,43 +17,6 @@ from mygpt import BracketedSequence ###################################################################### - -class Gang(nn.Module): - def __init__(self, models, nb_models_for_generation, mode="groupthink"): - super().__init__() - self.models = nn.ModuleList(models) - self.nb_models_for_generation = nb_models_for_generation - self.mode = mode - - def forward(self, bs): - # If first = 0, we are re-starting an auto-regressive process, - # that's the right moment to randomize who gonna do it - if bs.first == 0: - self.models_to_use = [ - self.models[k] - for k in torch.randperm(len(self.models))[ - : self.nb_models_for_generation - ] - ] - - all_the_logits = torch.cat( - [model(bs).x[None] for model in self.models_to_use], dim=0 - ) - - if self.mode == "groupthink": - y = all_the_logits.mean(dim=0) - elif self.mode == "groupwork": - m = torch.rand(all_the_logits.size(), device=all_the_logits.device) - m = (m.sort(dim=0).indices == 0).long() - y = (y * m).sum(dim=0) - else: - raise ValueError(f"Invalid mode {self.mode}") - - return BracketedSequence(y, bs.first, bs.nb) - - -###################################################################### - # ar_mask is a tensor with 0s and 1s, of same shape as input, with # 1s where tokens should be generated. The others are kept # unchanged. @@ -176,6 +139,7 @@ class QuizzMachine: self.train_w_quizzes = self.problem.generate_token_sequences( nb_train_samples ).to(device) + self.test_w_quizzes = self.problem.generate_token_sequences(nb_test_samples).to( device ) @@ -323,7 +287,9 @@ class QuizzMachine: return torch.cat([c_quizzes[:, l + 1 :], direction, c_quizzes[:, :l]], dim=1) - def comput_correctness(self, c_quizzes, models_for_validation): + def compute_correctness( + self, c_quizzes, models_for_validation, both_directions=True + ): reversed_c_quizzes = self.reverse_time(c_quizzes) ar_mask = self.make_ar_mask(c_quizzes) @@ -350,33 +316,36 @@ class QuizzMachine: correct = (c_quizzes == result).long().min(dim=-1).values - reversed_result = reversed_c_quizzes.clone() + if both_directions: + reversed_result = reversed_c_quizzes.clone() - masked_inplace_autoregression( - model=model, - batch_size=self.batch_size, - input=reversed_result, - ar_mask=ar_mask, - seq_logproba=seq_logproba, - temperature=1.0, - deterministic_synthesis=True, - # progress_bar_desc="solving reversed c_quizzes", - device=self.device, - ) + masked_inplace_autoregression( + model=model, + batch_size=self.batch_size, + input=reversed_result, + ar_mask=ar_mask, + seq_logproba=seq_logproba, + temperature=1.0, + deterministic_synthesis=True, + # progress_bar_desc="solving reversed c_quizzes", + device=self.device, + ) - reversed_correct = ( - (reversed_c_quizzes == reversed_result).long().min(dim=-1).values - ) + reversed_correct = ( + (reversed_c_quizzes == reversed_result).long().min(dim=-1).values + ) + + correct *= reversed_correct - nb_correct += correct * reversed_correct + # endif + + nb_correct += correct return nb_correct ############################################################### - def generate_quizzes( - self, nb, model_for_generation, min_ave_seq_logproba, reverse_cleanup=False - ): + def generate_quizzes(self, nb, model_for_generation, reverse_cleanup=False): c_quizzes = torch.empty( nb, self.train_w_quizzes.size(1), device=self.device, dtype=torch.int64 ) @@ -386,8 +355,11 @@ class QuizzMachine: ar_mask_solve = 1 - ar_mask_prompt seq_logproba = torch.empty(ar_mask_prompt.size(0), device=self.device) - warnings.warn("very high temperature with reversed cleanup", RuntimeWarning) - temperature = 10 + if reverse_cleanup: + warnings.warn("very high temperature with reversed cleanup", RuntimeWarning) + temperature = 10.0 + else: + temperature = 1.0 # warnings.warn("noise injection", RuntimeWarning) # noise_std = torch.rand(1).item() @@ -403,7 +375,6 @@ class QuizzMachine: seq_logproba=seq_logproba, temperature=temperature, deterministic_synthesis=False, - # progress_bar_desc="sampling c_quizzes", device=self.device, ) @@ -419,7 +390,6 @@ class QuizzMachine: seq_logproba=seq_logproba, temperature=temperature, deterministic_synthesis=True, - # progress_bar_desc="sampling c_quizzes", device=self.device, ) @@ -433,54 +403,19 @@ class QuizzMachine: seq_logproba=seq_logproba, temperature=temperature, deterministic_synthesis=True, - # progress_bar_desc="sampling c_quizzes", device=self.device, ) - return c_quizzes, seq_logproba.mean() - - ###################################################################### - - def create_c_quizzes( - self, - nb, - model_for_generation, - models_for_validation, - min_ave_seq_logproba, - reverse_cleanup, - n_epoch, - result_dir, - ): - c_quizzes, ave_seq_logproba = self.generate_quizzes( - nb, - model_for_generation=model_for_generation, - min_ave_seq_logproba=min_ave_seq_logproba, - reverse_cleanup=reverse_cleanup, - ) - - nb_correct = self.comput_correctness(c_quizzes, models_for_validation) - - return c_quizzes, nb_correct, ave_seq_logproba - - ###################################################################### + c_quizzes = self.reverse_time(c_quizzes) + masked_inplace_autoregression( + model=model_for_generation, + batch_size=self.batch_size, + input=c_quizzes, + ar_mask=ar_mask_solve, + seq_logproba=seq_logproba, + temperature=temperature, + deterministic_synthesis=True, + device=self.device, + ) - def gang_create_c_quizzes( - self, - nb, - nb_models_for_generation, - models, - mode, - min_ave_seq_logproba, - n_epoch, - result_dir, - ): - model_for_generation = Gang(models, nb_models_for_generation, mode) - models_for_validation = models - return self.create_c_quizzes( - nb, - model_for_generation, - models_for_validation, - min_ave_seq_logproba, - n_epoch, - result_dir, - ) + return c_quizzes, seq_logproba.mean()