Update.

[culture.git] / main.py

diff --git a/main.py b/main.py
index b88cbc4..b4ab473 100755 (executable)
--- a/main.py
+++ b/main.py
@@ -20,16 +20,6 @@ import threading
 
 import torch.multiprocessing as mp
 
-# world quizzes vs. culture quizzes
-
-######################################################################
-
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-    torch.backends.cuda.matmul.allow_tf32 = True
-else:
-    device = torch.device("cpu")
-
 ######################################################################
 
 parser = argparse.ArgumentParser(
@@ -82,7 +72,7 @@ parser.add_argument("--problem", type=str, default="grids")
 
 parser.add_argument("--nb_threads", type=int, default=1)
 
-parser.add_argument("--nb_gpus", type=int, default=1)
+parser.add_argument("--gpus", type=str, default="all")
 
 parser.add_argument("--nb_gpts", type=int, default=5)
 
@@ -92,12 +82,29 @@ parser.add_argument("--max_to_validate", type=int, default=None)
 
 parser.add_argument("--accuracy_to_make_c_quizzes", type=float, default=0.975)
 
+parser.add_argument("--proba_understands", type=float, default=0.99)
+
+parser.add_argument("--proba_not_understands", type=float, default=0.5)
+
 parser.add_argument("--generation_temperature", type=float, default=2.0)
 
 parser.add_argument("--dirty_debug", action="store_true", default=False)
 
 ######################################################################
 
+grids_tasks = ", ".join(
+    [x.__name__.removeprefix("task_") for x in grids.Grids().all_tasks]
+)
+
+parser.add_argument(
+    "--grids_tasks",
+    type=str,
+    default=None,
+    help="A comma-separated subset of: " + grids_tasks + ", or None for all.",
+)
+
+######################################################################
+
 parser.add_argument("--sky_height", type=int, default=6)
 
 parser.add_argument("--sky_width", type=int, default=8)
@@ -221,6 +228,19 @@ for n in vars(args):
 
 ######################################################################
 
+if args.gpus == "all":
+    gpus_idx = range(torch.cuda.device_count())
+else:
+    gpus_idx = [int(k) for k in args.gpus.split(",")]
+
+gpus = [torch.device(f"cuda:{n}") for n in gpus_idx]
+
+if torch.cuda.is_available():
+    main_device = gpus[0]
+else:
+    assert len(gpus) == 0
+    main_device = torch.device("cpu")
+
 if args.dirty_debug:
     args.nb_train_samples = 2500
     args.nb_test_samples = 100
@@ -240,16 +260,17 @@ if args.problem == "sky":
         nb_birds=args.sky_nb_birds,
         nb_iterations=args.sky_nb_iterations,
         speed=args.sky_speed,
-        max_nb_cached_chunks=args.nb_gpus * args.nb_train_samples // 100,
+        max_nb_cached_chunks=len(gpus) * args.nb_train_samples // 100,
         chunk_size=100,
         nb_threads=args.nb_threads,
     )
     back_accuracy = False
 elif args.problem == "grids":
     problem = grids.Grids(
-        max_nb_cached_chunks=args.nb_gpus * args.nb_train_samples // 100,
+        max_nb_cached_chunks=len(gpus) * args.nb_train_samples // 100,
         chunk_size=100,
         nb_threads=args.nb_threads,
+        tasks=args.grids_tasks,
     )
     back_accuracy = True
 else:
@@ -265,12 +286,12 @@ quiz_machine = quiz_machine.QuizMachine(
     batch_size=args.physical_batch_size,
     result_dir=args.result_dir,
     logger=log_string,
-    device=device,
+    device=main_device,
 )
 
 ######################################################################
 
-log_string(f"device {device}")
+log_string(f"main_device {main_device} gpus {[ str(g) for g in gpus]}")
 
 vocabulary_size = quiz_machine.vocabulary_size()
 
@@ -279,13 +300,7 @@ log_string(f"vocabulary_size {vocabulary_size}")
 ######################################################################
 
 
-######################################################################
-
-
-def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None):
-    if local_device is None:
-        local_device = device
-
+def run_tests(model, quiz_machine, deterministic_synthesis, local_device=main_device):
     with torch.autograd.no_grad():
         model.eval().to(local_device)
 
@@ -306,7 +321,7 @@ def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None):
 
         test_perplexity = math.exp(min(100, acc_test_loss / nb_test_samples))
 
-        log_string(f"test_perplexity {n_epoch} {test_perplexity}")
+        log_string(f"test_perplexity {n_epoch} model {model.id} {test_perplexity}")
 
         model.main_test_accuracy = quiz_machine.produce_results(
             n_epoch=n_epoch,
@@ -316,10 +331,7 @@ def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None):
         )
 
 
-def one_epoch(model, quiz_machine, local_device=None):
-    if local_device is None:
-        local_device = device
-
+def one_epoch(model, quiz_machine, local_device=main_device):
     optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
 
     model.to(local_device).train()
@@ -345,17 +357,21 @@ def one_epoch(model, quiz_machine, local_device=None):
 
     train_perplexity = math.exp(min(100, acc_train_loss / nb_train_samples))
 
-    log_string(f"train_perplexity {n_epoch} model.id {model.id} {train_perplexity}")
+    log_string(f"train_perplexity {n_epoch} model {model.id} {train_perplexity}")
 
     run_tests(model, quiz_machine, deterministic_synthesis=False)
 
+    model.to(main_device)
+
 
 ######################################################################
 
 
 def standard_validity(logproba):
     l = logproba.sort(dim=-1).values
-    return (l[:, 0] < math.log(0.5)) & (l[:, 1] > math.log(0.99))
+    return (l[:, 0] < math.log(args.proba_not_understands)) & (
+        l[:, 1] > math.log(args.proba_understands)
+    )
 
 
 def valid_c_quizzes(recorded, criteria):
@@ -445,7 +461,7 @@ for k in range(args.nb_gpts):
         nb_blocks=args.nb_blocks,
         causal=True,
         dropout=args.dropout,
-    ).to(device)
+    ).to(main_device)
 
     model.main_test_accuracy = 0.0
     model.id = k
@@ -544,15 +560,15 @@ for n_epoch in range(args.nb_epochs):
 
     ranked_models = sorted(models, key=lambda m: float(m.main_test_accuracy))
 
-    weakest_models = ranked_models[: args.nb_gpus]
+    weakest_models = ranked_models[: len(gpus)]
 
     threads = []
 
-    for gpu_id, model in enumerate(weakest_models):
+    for gpu, model in zip(gpus, weakest_models):
         log_string(f"training model {model.id}")
 
         t = threading.Thread(
-            target=one_epoch, daemon=True, args=(model, quiz_machine, f"cuda:{gpu_id}")
+            target=one_epoch, daemon=True, args=(model, quiz_machine, gpu)
         )
 
         threads.append(t)