X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;ds=sidebyside;f=main.py;h=63819f284225dc0c98669c367c5aa43b159c7ed2;hb=3ba9d1e0d85d689c2bdea9d2d571c6e8851a55b5;hp=5956be5f91effe796d5b984e0447ddc7c4c46e67;hpb=07c065e77f1d2a775814ec402752a4a8eb6c7574;p=culture.git diff --git a/main.py b/main.py index 5956be5..63819f2 100755 --- a/main.py +++ b/main.py @@ -18,15 +18,9 @@ import sky, grids, quiz_machine import threading -# world quizzes vs. culture quizzes - -###################################################################### +import torch.multiprocessing as mp -if torch.cuda.is_available(): - device = torch.device("cuda") - torch.backends.cuda.matmul.allow_tf32 = True -else: - device = torch.device("cpu") +# world quizzes vs. culture quizzes ###################################################################### @@ -80,7 +74,7 @@ parser.add_argument("--problem", type=str, default="grids") parser.add_argument("--nb_threads", type=int, default=1) -parser.add_argument("--nb_gpus", type=int, default=1) +parser.add_argument("--gpus", type=str, default="all") parser.add_argument("--nb_gpts", type=int, default=5) @@ -88,7 +82,11 @@ parser.add_argument("--min_to_validate", type=int, default=None) parser.add_argument("--max_to_validate", type=int, default=None) -parser.add_argument("--accuracy_to_make_c_quizzes", type=float, default=0.9) +parser.add_argument("--accuracy_to_make_c_quizzes", type=float, default=0.975) + +parser.add_argument("--proba_understands", type=float, default=0.99) + +parser.add_argument("--proba_not_understands", type=float, default=0.5) parser.add_argument("--generation_temperature", type=float, default=2.0) @@ -96,6 +94,19 @@ parser.add_argument("--dirty_debug", action="store_true", default=False) ###################################################################### +grids_tasks = ", ".join( + [x.__name__.removeprefix("task_") for x in grids.Grids().all_tasks] +) + +parser.add_argument( + "--grids_tasks", + type=str, + default=None, + help="A comma-separated subset of: " + grids_tasks + ", or None for all.", +) + +###################################################################### + parser.add_argument("--sky_height", type=int, default=6) parser.add_argument("--sky_width", type=int, default=8) @@ -219,6 +230,19 @@ for n in vars(args): ###################################################################### +if args.gpus == "all": + gpus_idx = range(torch.cuda.device_count()) +else: + gpus_idx = [int(k) for k in args.gpus.split(",")] + +gpus = [torch.device(f"cuda:{n}") for n in gpus_idx] + +if torch.cuda.is_available(): + main_device = gpus[0] +else: + assert len(gpus) == 0 + main_device = torch.device("cpu") + if args.dirty_debug: args.nb_train_samples = 2500 args.nb_test_samples = 100 @@ -238,16 +262,17 @@ if args.problem == "sky": nb_birds=args.sky_nb_birds, nb_iterations=args.sky_nb_iterations, speed=args.sky_speed, - max_nb_cached_chunks=args.nb_gpus * args.nb_train_samples // 100, + max_nb_cached_chunks=len(gpus) * args.nb_train_samples // 100, chunk_size=100, nb_threads=args.nb_threads, ) back_accuracy = False elif args.problem == "grids": problem = grids.Grids( - max_nb_cached_chunks=args.nb_gpus * args.nb_train_samples // 100, + max_nb_cached_chunks=len(gpus) * args.nb_train_samples // 100, chunk_size=100, nb_threads=args.nb_threads, + tasks=args.grids_tasks, ) back_accuracy = True else: @@ -263,12 +288,12 @@ quiz_machine = quiz_machine.QuizMachine( batch_size=args.physical_batch_size, result_dir=args.result_dir, logger=log_string, - device=device, + device=main_device, ) ###################################################################### -log_string(f"device {device}") +log_string(f"main_device {main_device} gpus {[ str(g) for g in gpus]}") vocabulary_size = quiz_machine.vocabulary_size() @@ -277,13 +302,7 @@ log_string(f"vocabulary_size {vocabulary_size}") ###################################################################### -###################################################################### - - -def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None): - if local_device is None: - local_device = device - +def run_tests(model, quiz_machine, deterministic_synthesis, local_device=main_device): with torch.autograd.no_grad(): model.eval().to(local_device) @@ -314,10 +333,7 @@ def run_tests(model, quiz_machine, deterministic_synthesis, local_device=None): ) -def one_epoch(model, quiz_machine, local_device=None): - if local_device is None: - local_device = device - +def one_epoch(model, quiz_machine, local_device=main_device): optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate) model.to(local_device).train() @@ -343,22 +359,19 @@ def one_epoch(model, quiz_machine, local_device=None): train_perplexity = math.exp(min(100, acc_train_loss / nb_train_samples)) - log_string(f"train_perplexity {n_epoch} {train_perplexity}") + log_string(f"train_perplexity {n_epoch} model.id {model.id} {train_perplexity}") run_tests(model, quiz_machine, deterministic_synthesis=False) - model.TRAINING_LOCK.release() - ###################################################################### def standard_validity(logproba): l = logproba.sort(dim=-1).values - return (l[:, 0] < math.log(0.5)) & (l[:, 1] > math.log(0.99)) - # warnings.warn("TEST!!!", RuntimeWarning) - # print(l.exp()) - # return (l[:, 0] < math.log(0.99)) + return (l[:, 0] < math.log(args.proba_not_understands)) & ( + l[:, 1] > math.log(args.proba_understands) + ) def valid_c_quizzes(recorded, criteria): @@ -448,19 +461,14 @@ for k in range(args.nb_gpts): nb_blocks=args.nb_blocks, causal=True, dropout=args.dropout, - ).to(device) + ).to(main_device) model.main_test_accuracy = 0.0 model.id = k - model.TRAINING_LOCK = threading.Lock() - model.train_w_quizzes = quiz_machine.generate_token_sequences( - args.nb_train_samples - ).to(device) + model.train_w_quizzes = quiz_machine.generate_token_sequences(args.nb_train_samples) quiz_machine.reverse_random_half_in_place(model.train_w_quizzes) - model.test_w_quizzes = quiz_machine.generate_token_sequences( - args.nb_test_samples - ).to(device) + model.test_w_quizzes = quiz_machine.generate_token_sequences(args.nb_test_samples) quiz_machine.reverse_random_half_in_place(model.test_w_quizzes) models.append(model) @@ -536,7 +544,7 @@ if args.dirty_debug: def standard_validity(logproba): l = logproba.sort(dim=-1).values - return l[:, 0] < math.log(0.99) + return l[:, 0] < math.log(0.5) ###################################################################### @@ -548,34 +556,37 @@ for n_epoch in range(args.nb_epochs): log_string(f"current_test_accuracies {cta}") ################################################## - # Select, improve, and eval the worst models + # Select, improve, and eval the worst model ranked_models = sorted(models, key=lambda m: float(m.main_test_accuracy)) - weakest_models = ranked_models[: args.nb_gpus] + weakest_models = ranked_models[: len(gpus)] + + threads = [] - for gpu_id, model in enumerate(weakest_models): - model.TRAINING_LOCK.acquire() + for gpu, model in zip(gpus, weakest_models): + log_string(f"training model {model.id}") - log_string( - f"training model {model.id} main_test_accuracy {model.main_test_accuracy}" + t = threading.Thread( + target=one_epoch, daemon=True, args=(model, quiz_machine, gpu) ) - threading.Thread( - target=one_epoch, daemon=True, args=(model, quiz_machine, f"cuda:{gpu_id}") - ).start() + threads.append(t) - for model in weakest_models: - model.TRAINING_LOCK.acquire() - model.TRAINING_LOCK.release() + t.start() + + for t in threads: + t.join() ################################################## - # Renew the train sets + # Replace a fraction of the w_quizzes with fresh ones log_string( f"cache_w_quizzes contains {quiz_machine.problem.nb_cached_quizzes()} quizzes" ) + # Renew entirely the train set + for model in weakest_models: quiz_machine.renew_w_quizzes(model, args.nb_train_samples)