X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=main.py;h=6c4099f51513c5e1550b3820466d303639c6e9d4;hb=5b7022591f48382ec84b1dda17297b1ed15166d7;hp=a8ceac8227bfab386ce34885ccbc00a687b4c442;hpb=2f87c91cf606a068de1450d198660de7e44cd356;p=culture.git diff --git a/main.py b/main.py index a8ceac8..6c4099f 100755 --- a/main.py +++ b/main.py @@ -78,10 +78,6 @@ parser.add_argument("--gpus", type=str, default="all") parser.add_argument("--nb_gpts", type=int, default=5) -parser.add_argument("--min_to_validate", type=int, default=None) - -parser.add_argument("--max_to_validate", type=int, default=None) - parser.add_argument("--accuracy_to_make_c_quizzes", type=float, default=0.975) parser.add_argument("--proba_understands", type=float, default=0.99) @@ -121,12 +117,6 @@ parser.add_argument("--sky_speed", type=int, default=3) args = parser.parse_args() -if args.min_to_validate is None: - args.min_to_validate = args.nb_gpts - 1 - -if args.max_to_validate is None: - args.max_to_validate = args.nb_gpts - 1 - if args.result_dir is None: args.result_dir = f"results_culture" @@ -226,6 +216,10 @@ def log_string(s): sys.stdout.flush() +now = time.strftime("%Y%m%d-%H%M%S", time.localtime()) + +os.system(f"tar zcvf {args.result_dir}/src-{now}.tgz *.py") + log_string(f"argv {' '.join(sys.argv)}") for n in vars(args): @@ -338,10 +332,10 @@ def run_tests(model, quiz_machine, deterministic_synthesis, local_device=main_de def one_epoch(model, quiz_machine, local_device=main_device): - optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate) - model.to(local_device).train() + optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate) + nb_train_samples, acc_train_loss = 0, 0.0 for input in quiz_machine.batches(model, split="train"): @@ -380,77 +374,94 @@ def standard_validity(logproba): ) -def valid_c_quizzes(recorded, criteria): - result = [q[criteria(lp)] for q, lp in recorded] - return torch.cat(result, dim=0) if len(result) > 0 else torch.tensor([]) +def valid_quizzes_and_logprobas(recorded, criteria): + validated_quizzes, validated_logprobas = [], [] + for q, lp in recorded: + validated_indices = criteria(lp) + validated_quizzes.append(q[validated_indices]) + validated_logprobas.append(lp[validated_indices]) + if len(validated_quizzes) > 0: + return torch.cat(validated_quizzes, dim=0), torch.cat( + validated_logprobas, dim=0 + ) + else: + return None, None -###################################################################### +###################################################################### -def create_c_quizzes( - models, - quiz_machine, - nb_for_train=1000, - nb_for_test=100, -): - quizzes_and_logproba_records = [] +def create_c_quizzes(models, quiz_machine, nb_for_train=1000, nb_for_test=100): nb_to_create = nb_for_train + nb_for_test - # ------------------------------------------------------------ + recorded_quizzes_logprobas = [] - file_name = os.path.join(args.result_dir, f"culture_c_quiz_{n_epoch:04d}_logp.dat") + nb_validated = 0 - with open(file_name, "w") as logp_file: - while ( - valid_c_quizzes(quizzes_and_logproba_records, standard_validity).size(0) - < nb_to_create - ): - # Select a model at random to generate the new quizzes + while nb_validated < nb_to_create: + model_for_generation = models[torch.randint(len(models), (1,))] - model_for_generation = models[torch.randint(len(models), (1,))] + c_quizzes = quiz_machine.generate_quizzes( + nb_to_create, + model_for_generation=model_for_generation, + temperature=args.generation_temperature, + ) - c_quizzes = quiz_machine.generate_quizzes( - nb_to_create, - model_for_generation=model_for_generation, - temperature=args.generation_temperature, - ) + c_quizzes = c_quizzes[quiz_machine.non_trivial(c_quizzes)] - c_quizzes = c_quizzes[quiz_machine.non_trivial(c_quizzes)] + if c_quizzes.size(0) > 0: + logproba = quiz_machine.logproba_of_solutions(models, c_quizzes) + recorded_quizzes_logprobas.append((c_quizzes, logproba)) - if c_quizzes.size(0) > 0: - logproba = quiz_machine.logproba_of_solutions(models, c_quizzes) - for l in logproba: - s = " ".join([str(x.item()) for x in l]) - logp_file.write(s + "\n") - quizzes_and_logproba_records.append((c_quizzes, logproba)) + validated_quizzes, validated_logprobas = valid_quizzes_and_logprobas( + recorded_quizzes_logprobas, standard_validity + ) - nb_validated = valid_c_quizzes( - quizzes_and_logproba_records, standard_validity - ).size(0) + if validated_quizzes is not None: + nb_validated = validated_quizzes.size(0) - log_string( - f"keep c_quizzes model {model_for_generation.id} nb_accumulated {nb_validated} / {nb_to_create}" - ) + log_string( + f"keep c_quizzes model {model_for_generation.id} nb_accumulated {nb_validated} / {nb_to_create}" + ) # store the new c_quizzes which have been validated - new_c_quizzes = valid_c_quizzes(quizzes_and_logproba_records, standard_validity) + quiz_machine.reverse_random_half_in_place(validated_quizzes) + quiz_machine.store_c_quizzes(validated_quizzes[:nb_for_train], for_train=True) + quiz_machine.store_c_quizzes( + validated_quizzes[nb_for_train:nb_to_create], for_train=False + ) - quiz_machine.reverse_random_half_in_place(new_c_quizzes) + ###################################################################### + # save the log probas - quiz_machine.store_c_quizzes(new_c_quizzes[:nb_for_train], for_train=True) - quiz_machine.store_c_quizzes(new_c_quizzes[nb_for_train:], for_train=False) + file_name = os.path.join( + args.result_dir, f"culture_c_quiz_all_{n_epoch:04d}_logp.dat" + ) - # save images + with open(file_name, "w") as logp_file: + for _, ll in recorded_quizzes_logprobas: + for l in ll: + s = " ".join([str(x.item()) for x in l]) + logp_file.write(s + "\n") - q = new_c_quizzes[:72] + ###################################################################### + # save images with their logprobas - if q.size(0) > 0: - quiz_machine.save_quiz_illustrations( - args.result_dir, f"culture_c_quiz_{n_epoch:04d}", q - ) + vq = validated_quizzes[:72] + vl = validated_logprobas[:72] + + if vq.size(0) > 0: + prefix = f"culture_c_quiz_{n_epoch:04d}" + + file_name = os.path.join(args.result_dir, prefix + "_logp.dat") + with open(file_name, "w") as logp_file: + for l in vl: + s = " ".join([str(x.item()) for x in l]) + logp_file.write(s + "\n") + + quiz_machine.save_quiz_illustrations(args.result_dir, prefix, vq) ###################################################################### @@ -488,9 +499,9 @@ if args.resume: filename = f"gpt_{model.id:03d}.pth" try: - model.load_state_dict( - torch.load(os.path.join(args.result_dir, filename)) - ) + d = torch.load(os.path.join(args.result_dir, filename)) + model.load_state_dict(d[0]) + model.main_test_accuracy = d[1] log_string(f"successfully loaded {filename}") except FileNotFoundError: log_string(f"cannot find {filename}") @@ -614,19 +625,17 @@ for n_epoch in range(args.nb_epochs): for t in threads: t.join() + # Save the models to disk + for model in weakest_models: filename = f"gpt_{model.id:03d}.pth" - torch.save(model.state_dict(), os.path.join(args.result_dir, filename)) + torch.save( + (model.state_dict(), model.main_test_accuracy), + os.path.join(args.result_dir, filename), + ) log_string(f"wrote {filename}") - ################################################## - # Replace a fraction of the w_quizzes with fresh ones - - log_string( - f"cache_w_quizzes contains {quiz_machine.problem.nb_cached_quizzes()} quizzes" - ) - - # Renew entirely the train set + # Renew the training samples for model in weakest_models: quiz_machine.renew_w_quizzes(model, args.nb_train_samples) @@ -643,6 +652,8 @@ for n_epoch in range(args.nb_epochs): nb_for_test=nb_new_c_quizzes_for_test, ) - quiz_machine.save_c_quizzes(os.path.join(args.result_dir, "c_quizzes.pth")) + filename = "c_quizzes.pth" + quiz_machine.save_c_quizzes(os.path.join(args.result_dir, filename)) + log_string(f"wrote {filename}") ######################################################################