X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;ds=sidebyside;f=main.py;h=b4ab473abb8e6f13c55265e8b8736d2f10856cac;hb=dc28a8c5cc4456cd9512c57947b104ac23338c8f;hp=73e7ca2501a030d2b70819f42b648b59ff981b8f;hpb=1e7e3d3877ae98737852b78f35a98030dcc0701f;p=culture.git diff --git a/main.py b/main.py index 73e7ca2..b4ab473 100755 --- a/main.py +++ b/main.py @@ -18,15 +18,7 @@ import sky, grids, quiz_machine import threading -# world quizzes vs. culture quizzes - -###################################################################### - -if torch.cuda.is_available(): - device = torch.device("cuda") - torch.backends.cuda.matmul.allow_tf32 = True -else: - device = torch.device("cpu") +import torch.multiprocessing as mp ###################################################################### @@ -80,7 +72,7 @@ parser.add_argument("--problem", type=str, default="grids") parser.add_argument("--nb_threads", type=int, default=1) -parser.add_argument("--nb_gpus", type=int, default=1) +parser.add_argument("--gpus", type=str, default="all") parser.add_argument("--nb_gpts", type=int, default=5) @@ -90,12 +82,29 @@ parser.add_argument("--max_to_validate", type=int, default=None) parser.add_argument("--accuracy_to_make_c_quizzes", type=float, default=0.975) +parser.add_argument("--proba_understands", type=float, default=0.99) + +parser.add_argument("--proba_not_understands", type=float, default=0.5) + parser.add_argument("--generation_temperature", type=float, default=2.0) parser.add_argument("--dirty_debug", action="store_true", default=False) ###################################################################### +grids_tasks = ", ".join( + [x.__name__.removeprefix("task_") for x in grids.Grids().all_tasks] +) + +parser.add_argument( + "--grids_tasks", + type=str, + default=None, + help="A comma-separated subset of: " + grids_tasks + ", or None for all.", +) + +###################################################################### + parser.add_argument("--sky_height", type=int, default=6) parser.add_argument("--sky_width", type=int, default=8) @@ -219,6 +228,19 @@ for n in vars(args): ###################################################################### +if args.gpus == "all": + gpus_idx = range(torch.cuda.device_count()) +else: + gpus_idx = [int(k) for k in args.gpus.split(",")] + +gpus = [torch.device(f"cuda:{n}") for n in gpus_idx] + +if torch.cuda.is_available(): + main_device = gpus[0] +else: + assert len(gpus) == 0 + main_device = torch.device("cpu") + if args.dirty_debug: args.nb_train_samples = 2500 args.nb_test_samples = 100 @@ -238,21 +260,24 @@ if args.problem == "sky": nb_birds=args.sky_nb_birds, nb_iterations=args.sky_nb_iterations, speed=args.sky_speed, - max_nb_cached_chunks=args.nb_gpus * args.nb_train_samples // 100, + max_nb_cached_chunks=len(gpus) * args.nb_train_samples // 100, chunk_size=100, nb_threads=args.nb_threads, ) back_accuracy = False elif args.problem == "grids": problem = grids.Grids( - max_nb_cached_chunks=args.nb_gpus * args.nb_train_samples // 100, + max_nb_cached_chunks=len(gpus) * args.nb_train_samples // 100, chunk_size=100, nb_threads=args.nb_threads, + tasks=args.grids_tasks, ) back_accuracy = True else: raise ValueError +problem.save_some_examples(args.result_dir) + quiz_machine = quiz_machine.QuizMachine( problem=problem, nb_train_samples=args.nb_train_samples, @@ -261,12 +286,12 @@ quiz_machine = quiz_machine.QuizMachine( batch_size=args.physical_batch_size, result_dir=args.result_dir, logger=log_string, - device=device, + device=main_device, ) ###################################################################### -log_string(f"device {device}") +log_string(f"main_device {main_device} gpus {[ str(g) for g in gpus]}") vocabulary_size = quiz_machine.vocabulary_size() @@ -275,13 +300,7 @@ log_string(f"vocabulary_size {vocabulary_size}") ###################################################################### -###################################################################### - - -def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None): - if local_device is None: - local_device = device - +def run_tests(model, quiz_machine, deterministic_synthesis, local_device=main_device): with torch.autograd.no_grad(): model.eval().to(local_device) @@ -302,7 +321,7 @@ def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None): test_perplexity = math.exp(min(100, acc_test_loss / nb_test_samples)) - log_string(f"test_perplexity {n_epoch} {test_perplexity}") + log_string(f"test_perplexity {n_epoch} model {model.id} {test_perplexity}") model.main_test_accuracy = quiz_machine.produce_results( n_epoch=n_epoch, @@ -312,10 +331,7 @@ def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None): ) -def one_epoch(model, quiz_machine, local_device=None): - if local_device is None: - local_device = device - +def one_epoch(model, quiz_machine, local_device=main_device): optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate) model.to(local_device).train() @@ -341,11 +357,11 @@ def one_epoch(model, quiz_machine, local_device=None): train_perplexity = math.exp(min(100, acc_train_loss / nb_train_samples)) - log_string(f"train_perplexity {n_epoch} {train_perplexity}") + log_string(f"train_perplexity {n_epoch} model {model.id} {train_perplexity}") run_tests(model, quiz_machine, deterministic_synthesis=False) - model.TRAINING_LOCK.release() + model.to(main_device) ###################################################################### @@ -353,10 +369,9 @@ def one_epoch(model, quiz_machine, local_device=None): def standard_validity(logproba): l = logproba.sort(dim=-1).values - return (l[:, 0] < math.log(0.5)) & (l[:, 1] > math.log(0.99)) - # warnings.warn("TEST!!!", RuntimeWarning) - # print(l.exp()) - # return (l[:, 0] < math.log(0.99)) + return (l[:, 0] < math.log(args.proba_not_understands)) & ( + l[:, 1] > math.log(args.proba_understands) + ) def valid_c_quizzes(recorded, criteria): @@ -446,19 +461,14 @@ for k in range(args.nb_gpts): nb_blocks=args.nb_blocks, causal=True, dropout=args.dropout, - ).to(device) + ).to(main_device) model.main_test_accuracy = 0.0 model.id = k - model.TRAINING_LOCK = threading.Lock() - model.train_w_quizzes = quiz_machine.generate_token_sequences( - args.nb_train_samples - ).to(device) + model.train_w_quizzes = quiz_machine.generate_token_sequences(args.nb_train_samples) quiz_machine.reverse_random_half_in_place(model.train_w_quizzes) - model.test_w_quizzes = quiz_machine.generate_token_sequences( - args.nb_test_samples - ).to(device) + model.test_w_quizzes = quiz_machine.generate_token_sequences(args.nb_test_samples) quiz_machine.reverse_random_half_in_place(model.test_w_quizzes) models.append(model) @@ -532,6 +542,11 @@ if args.dirty_debug: nb_new_c_quizzes_for_train = 100 nb_new_c_quizzes_for_test = 10 + def standard_validity(logproba): + l = logproba.sort(dim=-1).values + return l[:, 0] < math.log(0.5) + + ###################################################################### for n_epoch in range(args.nb_epochs): @@ -545,22 +560,23 @@ for n_epoch in range(args.nb_epochs): ranked_models = sorted(models, key=lambda m: float(m.main_test_accuracy)) - weakest_models = ranked_models[: args.nb_gpus] + weakest_models = ranked_models[: len(gpus)] - for gpu_id, model in enumerate(weakest_models): - model.TRAINING_LOCK.acquire() + threads = [] - log_string( - f"training model {model.id} main_test_accuracy {model.main_test_accuracy}" + for gpu, model in zip(gpus, weakest_models): + log_string(f"training model {model.id}") + + t = threading.Thread( + target=one_epoch, daemon=True, args=(model, quiz_machine, gpu) ) - threading.Thread( - target=one_epoch, daemon=True, args=(model, quiz_machine, f"cuda:{gpu_id}") - ).start() + threads.append(t) - for model in weakest_models: - model.TRAINING_LOCK.acquire() - model.TRAINING_LOCK.release() + t.start() + + for t in threads: + t.join() ################################################## # Replace a fraction of the w_quizzes with fresh ones