Update.

[culture.git] / main.py

diff --git a/main.py b/main.py
index 9c3d7f1..aefc3a1 100755 (executable)
--- a/main.py
+++ b/main.py
@@ -15,7 +15,6 @@ import ffutils
 
 import mygpt
 import sky, grids, quiz_machine
-from problem import MultiThreadProblem
 
 # world quizzes vs. culture quizzes
 
@@ -78,7 +77,7 @@ parser.add_argument("--deterministic_synthesis", action="store_true", default=Fa
 
 parser.add_argument("--problem", type=str, default="grids")
 
-parser.add_argument("--multi_thread_problem", action="store_true", default=False)
+parser.add_argument("--nb_threads", type=int, default=-1)
 
 parser.add_argument("--nb_gpts", type=int, default=5)
 
@@ -125,7 +124,7 @@ if args.result_dir is None:
 
 default_args = {
     "model": "37M",
-    "batch_size": 100,
+    "batch_size": 25,
     "nb_train_samples": 100000,
     "nb_test_samples": 10000,
 }
@@ -240,17 +239,22 @@ if args.problem == "sky":
         nb_birds=args.sky_nb_birds,
         nb_iterations=args.sky_nb_iterations,
         speed=args.sky_speed,
+        max_nb_cached_chunks=args.nb_train_samples // 100,
+        chunk_size=100,
+        nb_threads=args.nb_threads,
     )
     back_accuracy = False
 elif args.problem == "grids":
-    problem = grids.Grids(device=device)
+    problem = grids.Grids(
+        device=device,
+        max_nb_cached_chunks=args.nb_train_samples // 100,
+        chunk_size=100,
+        nb_threads=args.nb_threads,
+    )
     back_accuracy = True
 else:
     raise ValueError
 
-if args.multi_thread_problem:
-    problem = MultiThreadProblem(problem, args.nb_train_samples, chunk_size=1000)
-
 quiz_machine = quiz_machine.QuizMachine(
     problem=problem,
     nb_train_samples=args.nb_train_samples,
@@ -387,8 +391,13 @@ def run_tests(model, quiz_machine, deterministic_synthesis):
 ######################################################################
 
 
+def standard_validity(logproba):
+    l = logproba.sort(dim=-1).values
+    return logical_and(l[0] < math.log(0.5), l[1] > math.log(0.95))
+
+
 def valid_c_quizzes(recorded, criteria):
-    result = [q[criteria(c)] for q, c in recorded]
+    result = [q[criteria(lp)] for q, lp in recorded]
     return torch.cat(result, dim=0) if len(result) > 0 else torch.tensor([])
 
 
@@ -400,6 +409,80 @@ def create_c_quizzes(
     quiz_machine,
     nb_for_train=1000,
     nb_for_test=100,
+):
+    quizzes_and_logproba_records = []
+
+    nb_to_create = nb_for_train + nb_for_test
+
+    # ------------------------------------------------------------
+
+    file_name = os.path.join(args.result_dir, f"culture_c_quiz_{n_epoch:04d}_logp.dat")
+
+    with open(file_name, "w") as logp_file:
+        while (
+            valid_c_quizzes(quizzes_and_logproba_records, standard_validity).size(0)
+            < nb_to_create
+        ):
+            # Select a model at random to generate the new quizzes
+
+            model_for_generation = models[torch.randint(len(models), (1,))]
+
+            c_quizzes = quiz_machine.generate_quizzes(
+                nb_to_create,
+                model_for_generation=model_for_generation,
+                temperature=args.generation_temperature,
+            )
+
+            c_quizzes = c_quizzes[quiz_machine.non_trivial(c_quizzes)]
+
+            if c_quizzes.size(0) > 0:
+                logproba = c_quizzes.new(c_quizzes.size(0), len(models))
+                for q, l in zip(
+                    c_quizzes.split(args.batch_size), logits.split(args.batch_size)
+                ):
+                    for model in models:
+                        l[model.id] = F.cross_entropy(model(q))
+
+                for l in logproba:
+                    s = " ".join([str(x.item()) for x in l])
+                    logp_file.write(s + "\n")
+
+                quizzes_and_logproba_records.append((c_quizzes, logproba))
+
+            nb_validated = valid_c_quizzes(
+                quizzes_and_logproba_records, standard_validity
+            ).size(0)
+
+            log_string(
+                f"keep c_quizzes model {model_for_generation.id} nb_accumulated {nb_validated} / {nb_to_create}"
+            )
+
+    # store the new c_quizzes which have been validated
+
+    new_c_quizzes = valid_c_quizzes(quizzes_and_logproba_records, standard_validity)
+
+    quiz_machine.reverse_random_half_in_place(new_c_quizzes)
+
+    quiz_machine.store_c_quizzes(new_c_quizzes[:nb_for_train], for_train=True)
+    quiz_machine.store_c_quizzes(new_c_quizzes[nb_for_train:], for_train=False)
+
+    # save a bunch of images to investigate what quizzes with a
+    # certain nb of correct predictions look like
+
+    q = new_c_quizzes[:72]
+
+    if q.size(0) > 0:
+        quiz_machine.save_quizzes(args.result_dir, f"culture_c_quiz_{n_epoch:04d}", q)
+
+
+######################################################################
+
+
+def create_c_quizzes_(
+    models,
+    quiz_machine,
+    nb_for_train=1000,
+    nb_for_test=100,
 ):
     quizzes_and_nb_correct_records = []
 
@@ -428,6 +511,14 @@ def create_c_quizzes(
                 temperature=args.generation_temperature,
             )
 
+            # if args.prediction_correctness:
+
+            # else:
+            # logproba = quiz_machine.new(quiz_machine.size(0), len(models))
+            # for q,l in zip(quizzes.split(args.batch_size), logits.split(args.batch_size)):
+            # for model in models:
+            # l[...] = F.cross_entropy(model(q))
+
             c_quizzes = c_quizzes[quiz_machine.non_trivial(c_quizzes)]
 
             if c_quizzes.size(0) > 0: