Update.

[culture.git] / quizz_machine.py

diff --git a/quizz_machine.py b/quizz_machine.py
index 6e57fb4..62ae8ce 100755 (executable)
--- a/quizz_machine.py
+++ b/quizz_machine.py
@@ -122,12 +122,13 @@ class QuizzMachine:
         forward_to_backward = torch.cat(
             [
                 quizzes[:, 0:1],
-                quizzes[:, 2 + self.prompt_len :],
-                quizzes[:, 1 + self.prompt_len : 2 + self.prompt_len],
+                quizzes[:, 2 + self.prompt_len : 2 + self.prompt_len + self.answer_len],
+                quizzes[:, 1 + self.prompt_len : 1 + self.prompt_len + 1],
                 quizzes[:, 1 : 1 + self.prompt_len],
             ],
             dim=1,
         )
+
         forward_to_backward[:, 0] = self.token_backward
         forward_to_backward[:, 1 + self.answer_len] = self.token_backward
 
@@ -234,14 +235,14 @@ class QuizzMachine:
 
         if result_dir is not None:
             self.save_quizzes(
-                result_dir, "culture_w_quizzes", self.train_w_quizzes[:72]
+                result_dir,
+                "culture_w_quizzes",
+                self.train_w_quizzes[:72],
+                prediction=True,
             )
 
-            # toto = self.reverse_time(self.train_w_quizzes[:72])
-            # self.save_quizzes(result_dir, "toto", toto)
-            # exit(0)
-
     def save_quizzes(self, result_dir, filename_prefix, quizzes, prediction=False):
+        quizzes = quizzes.clone()
         forward = quizzes[quizzes[:, 0] == self.token_forward]
         ib = quizzes[:, 0] == self.token_backward
         backward = quizzes[ib]
@@ -327,6 +328,14 @@ class QuizzMachine:
             )
 
             if self.back_accuracy:
+                # If back_accuracy is True, we compute the accuracy on
+                # the backward quizzes not by counting how many time
+                # the real prompt A is equal to the reconstructed
+                # prompt A*, but how many time the answers B* computed
+                # from A* is equal to the correct answer. So we look
+                # for the accuracy of A->B*=B for the forward, but for
+                # the backward we look at B->A*->B*=B instead of B->A*=A
+
                 n_forward = input[:, 0] == self.token_forward
                 nb_total = input[n_forward].size(0)
                 nb_correct = (
@@ -334,17 +343,32 @@ class QuizzMachine:
                     .long()
                     .min(dim=1)
                     .values.sum()
+                    .item()
                 )
 
                 n_backward = input[:, 0] == self.token_backward
                 back_input = self.reverse_time(result[n_backward])
+
                 if back_input.size(0) > 0:
                     back_input[:, 2 + self.prompt_len :] = input[
-                        n_backward, 2 + self.prompt_len :
+                        n_backward, 1 : 1 + self.answer_len
                     ]
                     back_nb_total, back_nb_correct = compute_accuracy(back_input)
+
+                    self.logger(
+                        f"accuracy {n_epoch=} {model.id=} {nb_correct} / {nb_total}"
+                    )
+                    self.logger(
+                        f"back_accuracy {n_epoch=} {model.id=} {back_nb_correct} / {back_nb_total}"
+                    )
+
                     nb_total += back_nb_total
                     nb_correct += back_nb_correct
+                else:
+                    self.logger(
+                        f"accuracy {n_epoch=} {model.id=} {nb_correct} / {nb_total}"
+                    )
+
             else:
                 nb_total = input.size(0)
                 nb_correct = (input == result).long().min(dim=1).values.sum()