Update.

[culture.git] / main.py

diff --git a/main.py b/main.py
index 7f9d521..d0de5af 100755 (executable)
--- a/main.py
+++ b/main.py
@@ -58,7 +58,7 @@ parser.add_argument("--nb_train_samples", type=int, default=None)
 
 parser.add_argument("--nb_test_samples", type=int, default=None)
 
-parser.add_argument("--learning_rate", type=float, default=1e-4)
+parser.add_argument("--learning_rate", type=float, default=1e-3)
 
 ########################################
 
@@ -103,7 +103,7 @@ if args.dirty_debug:
 default_args = {
     "model": "37M",
     "batch_size": 100,
-    "nb_train_samples": 250000,
+    "nb_train_samples": 100000,
     "nb_test_samples": 10000,
 }
 
@@ -355,12 +355,19 @@ def create_c_quizzes(
     nb_for_test=100,
     min_ave_seq_logproba=None,
 ):
-    kept = []
+    # We will store the generated quizzes for each number of
+    # correct prediction
+    recorded = dict([(n, []) for n in range(len(models) + 1)])
+
     model_indexes = []
     sum_logits, sum_nb_c_quizzes = 0, 0
+    nb_correct_to_validate = len(models) - 1
 
-    while sum([x.size(0) for x in kept]) < nb_for_train + nb_for_test:
-        nb_to_generate = nb_for_train + nb_for_test
+    while (
+        sum([x.size(0) for x in recorded[nb_correct_to_validate]])
+        < nb_for_train + nb_for_test
+    ):
+        nb_to_validate = nb_for_train + nb_for_test
 
         if len(model_indexes) == 0:
             model_indexes = [i.item() for i in torch.randperm(len(models))]
@@ -368,7 +375,7 @@ def create_c_quizzes(
         model = models[model_indexes.pop()]
 
         new_c_quizzes, nb_correct, ave_seq_logproba = quizz_machine.create_c_quizzes(
-            nb=nb_to_generate,
+            nb=nb_to_validate,
             model_for_generation=model,
             models_for_validation=models,
             min_ave_seq_logproba=min_ave_seq_logproba,
@@ -380,30 +387,44 @@ def create_c_quizzes(
         sum_logits += new_c_quizzes.size(0) * ave_seq_logproba
         sum_nb_c_quizzes += new_c_quizzes.size(0)
 
-        to_keep = new_c_quizzes[nb_correct == len(models) - 1]
-
         if args.dirty_debug:
-            to_keep = new_c_quizzes[
-                torch.randint(3, (new_c_quizzes.size(0),), device=new_c_quizzes.device)
-                == 0
-            ]
+            nb_correct = torch.randint(
+                len(models) + 1, nb_correct.size(), device=new_c_quizzes.device
+            )
+
+        for n in range(nb_correct.max() + 1):
+            recorded[n].append(new_c_quizzes[nb_correct == n].clone())
 
-        kept.append(to_keep)
+        nb_validated = sum([x.size(0) for x in recorded[nb_correct_to_validate]])
+        nb_generated = sum(
+            [sum([x.size(0) for x in recorded[n]]) for n in recorded.keys()]
+        )
 
         log_string(
-            f"keep c_quizzes {to_keep.size(0)}/{new_c_quizzes.size(0)} ({to_keep.size(0)*100/new_c_quizzes.size(0):.02f}%) total {sum([ x.size(0) for x in kept])}/{nb_to_generate}"
+            f"keep c_quizzes {nb_validated*100/nb_generated:.02f}% kept total {nb_validated}/{nb_to_validate}"
         )
 
-    new_c_quizzes = torch.cat(kept, dim=0)[: nb_for_train + nb_for_test]
+    # concatenate and shuffle
+    for n in recorded.keys():
+        if len(recorded[n]) > 0:
+            q = torch.cat(recorded[n], dim=0)
+            q = q[torch.randperm(q.size(0), device=q.device)]
+            recorded[n] = q
+        else:
+            del recorded[n]
+
+    new_c_quizzes = recorded[nb_correct_to_validate][: nb_for_train + nb_for_test]
 
     quizz_machine.store_c_quizzes(new_c_quizzes[:nb_for_train], for_train=True)
     quizz_machine.store_c_quizzes(new_c_quizzes[nb_for_train:], for_train=False)
 
-    quizz_machine.problem.save_quizzes(
-        new_c_quizzes[:72],
-        args.result_dir,
-        f"culture_c_quiz_{n_epoch:04d}_{model.id:02d}",
-    )
+    for n in recorded.keys():
+        s = "_validated" if n == nb_correct_to_validate else ""
+        quizz_machine.problem.save_quizzes(
+            recorded[n][:72],
+            args.result_dir,
+            f"culture_c_quiz_{n_epoch:04d}_N{n}{s}",
+        )
 
     return sum_logits / sum_nb_c_quizzes
 
@@ -478,12 +499,12 @@ for n_epoch in range(args.nb_epochs):
         )
 
         # We keep the first average logits as a reference
-        if min_ave_seq_logproba is None:
-            min_ave_seq_logproba = ave_seq_logproba
-        else:
-            log_string(
-                f"min_ave_seq_logproba {min_ave_seq_logproba} ave_seq_logproba {ave_seq_logproba}"
-            )
+        # if min_ave_seq_logproba is None:
+        # min_ave_seq_logproba = ave_seq_logproba
+        # else:
+        # log_string(
+        # f"min_ave_seq_logproba {min_ave_seq_logproba} ave_seq_logproba {ave_seq_logproba}"
+        # )
 
         # We update everyone
         for model in models: