From: François Fleuret <francois@fleuret.org>
Date: Thu, 13 Jun 2024 17:42:51 +0000 (+0200)
Subject: Update.
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=d95b9b72b0f098b5c955395905a0aff710f553a7;p=culture.git

Update.
---

diff --git a/main.py b/main.py
index dace5f2..37515b5 100755
--- a/main.py
+++ b/main.py
@@ -815,7 +815,7 @@ for n_epoch in range(nb_epochs_finished, args.nb_epochs):
 
     model.train()
 
-    nb_train_samples, acc_train_loss_ar, acc_train_loss_ae = 0, 0.0, 0.0
+    nb_train_samples, acc_train_loss = 0, 0.0
 
     for input in task.batches(split="train"):
         input = input.to(device)
@@ -823,22 +823,13 @@ for n_epoch in range(nb_epochs_finished, args.nb_epochs):
         if nb_train_samples % args.batch_size == 0:
             optimizer.zero_grad()
 
-        if args.autoencoder_weight > 0:
-            bs_ar, bs_ae = model(mygpt.BracketedSequence(input), autoencoder=True)
-            output_ar, output_ae = bs_ar.x, bs_ae.x
-            loss_ar = F.cross_entropy(output_ar.transpose(1, 2), input)
-            loss_ae = F.cross_entropy(output_ae[:, 1:].transpose(1, 2), input[:, :-1])
-        else:
-            output = model(mygpt.BracketedSequence(input)).x
-            loss_ar = F.cross_entropy(output.transpose(1, 2), input)
-            loss_ae = loss_ar.new_full((1,), 0.0)
-
-        acc_train_loss_ar += loss_ar.item() * input.size(0)
-        acc_train_loss_ae += loss_ae.item() * input.size(0)
+        output = model(mygpt.BracketedSequence(input)).x
+        loss = F.cross_entropy(output.transpose(1, 2), input)
+        acc_train_loss += loss.item() * input.size(0)
 
         nb_train_samples += input.size(0)
 
-        (loss_ar + args.autoencoder_weight * loss_ae).backward()
+        loss.backward()
 
         if nb_train_samples % args.batch_size == 0:
             optimizer.step()
@@ -846,46 +837,28 @@ for n_epoch in range(nb_epochs_finished, args.nb_epochs):
     with torch.autograd.no_grad():
         model.eval()
 
-        nb_test_samples, acc_test_loss_ar, acc_test_loss_ae = 0, 0.0, 0.0
+        nb_test_samples, acc_test_loss = 0, 0.0
         nb_samples_accumulated = 0
 
         for input in task.batches(split="test"):
             input = input.to(device)
 
-            if args.autoencoder_weight > 0:
-                bs_ar, bs_ae = model(mygpt.BracketedSequence(input), autoencoder=True)
-                output_ar, output_ae = bs_ar.x, bs_ae.x
-                loss_ae = F.cross_entropy(
-                    output_ae[:, 1:].transpose(1, 2), input[:, :-1]
-                )
-                acc_test_loss_ae += loss_ae.item() * input.size(0)
-            else:
-                bs_ar = model(mygpt.BracketedSequence(input))
-                output_ar = bs_ar.x
+            bs = model(mygpt.BracketedSequence(input))
+            output_ar = bs.x
 
-            loss_ar = F.cross_entropy(output_ar.transpose(1, 2), input)
+            loss = F.cross_entropy(output.transpose(1, 2), input)
 
-            acc_test_loss_ar += loss_ar.item() * input.size(0)
+            acc_test_loss += loss.item() * input.size(0)
 
             nb_test_samples += input.size(0)
 
-        train_ar_perplexity = math.exp(min(100, acc_train_loss_ar / nb_train_samples))
-        test_ar_perplexity = math.exp(min(100, acc_test_loss_ar / nb_test_samples))
+        train_perplexity = math.exp(min(100, acc_train_loss / nb_train_samples))
+        test_perplexity = math.exp(min(100, acc_test_loss / nb_test_samples))
 
         log_string(
-            f"perplexity_ar {n_epoch} train_set {train_set_perplexity} train_prediction {train_ar_perplexity} test_prediction {test_ar_perplexity}"
+            f"perplexity {n_epoch} train_set {train_set_perplexity} train_prediction {train_perplexity} test_prediction {test_perplexity}"
         )
 
-        if args.autoencoder_weight > 0:
-            train_ae_perplexity = math.exp(
-                min(100, acc_train_loss_ae / nb_train_samples)
-            )
-            test_ae_perplexity = math.exp(min(100, acc_test_loss_ae / nb_test_samples))
-
-            log_string(
-                f"perplexity_ae {n_epoch} train_set {train_set_perplexity} train_prediction {train_ae_perplexity} test_prediction {test_ae_perplexity}"
-            )
-
         task.produce_results(
             n_epoch=n_epoch,
             model=model,