projects
/
culture.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Update.
[culture.git]
/
main.py
diff --git
a/main.py
b/main.py
index
a7338c7
..
b88cbc4
100755
(executable)
--- a/
main.py
+++ b/
main.py
@@
-255,6
+255,8
@@
elif args.problem == "grids":
else:
raise ValueError
else:
raise ValueError
+problem.save_some_examples(args.result_dir)
+
quiz_machine = quiz_machine.QuizMachine(
problem=problem,
nb_train_samples=args.nb_train_samples,
quiz_machine = quiz_machine.QuizMachine(
problem=problem,
nb_train_samples=args.nb_train_samples,
@@
-347,8
+349,6
@@
def one_epoch(model, quiz_machine, local_device=None):
run_tests(model, quiz_machine, deterministic_synthesis=False)
run_tests(model, quiz_machine, deterministic_synthesis=False)
- model.TRAINING_LOCK.release()
-
######################################################################
######################################################################
@@
-449,7
+449,6
@@
for k in range(args.nb_gpts):
model.main_test_accuracy = 0.0
model.id = k
model.main_test_accuracy = 0.0
model.id = k
- model.TRAINING_LOCK = threading.Lock()
model.train_w_quizzes = quiz_machine.generate_token_sequences(args.nb_train_samples)
quiz_machine.reverse_random_half_in_place(model.train_w_quizzes)
model.train_w_quizzes = quiz_machine.generate_token_sequences(args.nb_train_samples)
quiz_machine.reverse_random_half_in_place(model.train_w_quizzes)
@@
-547,20
+546,21
@@
for n_epoch in range(args.nb_epochs):
weakest_models = ranked_models[: args.nb_gpus]
weakest_models = ranked_models[: args.nb_gpus]
+ threads = []
+
for gpu_id, model in enumerate(weakest_models):
for gpu_id, model in enumerate(weakest_models):
-
model.TRAINING_LOCK.acquire(
)
+
log_string(f"training model {model.id}"
)
-
log_string
(
- f"training model {model.id} main_test_accuracy {model.main_test_accuracy}"
+
t = threading.Thread
(
+ target=one_epoch, daemon=True, args=(model, quiz_machine, f"cuda:{gpu_id}")
)
)
- threading.Thread(
- target=one_epoch, daemon=True, args=(model, quiz_machine, f"cuda:{gpu_id}")
- ).start()
+ threads.append(t)
- for model in weakest_models:
- model.TRAINING_LOCK.acquire()
- model.TRAINING_LOCK.release()
+ t.start()
+
+ for t in threads:
+ t.join()
##################################################
# Replace a fraction of the w_quizzes with fresh ones
##################################################
# Replace a fraction of the w_quizzes with fresh ones