From: François Fleuret <francois@fleuret.org>
Date: Thu, 23 Mar 2023 18:11:47 +0000 (+0100)
Subject: Update
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=88bcf05864ddf89d071ee4be17af57b3b3ce7c2a;p=beaver.git

Update
---

diff --git a/beaver.py b/beaver.py
index 86008f6..6a6343d 100755
--- a/beaver.py
+++ b/beaver.py
@@ -131,10 +131,10 @@ for n in vars(args):
 ######################################################################
 
 
-def generation_order(x, fixed_len):
+def generation_order(x, fixed_len=0):
     if args.random_regression_order:
         order = torch.rand(x.size(), device=x.device)
-        order[:, :fixed_len] = torch.linspace(-2, -1, fixed_len, device=x.device)
+        order[:, :fixed_len] = torch.arange(-fixed_len, 0, device=x.device)
         order = order.sort(1).indices
     else:
         order = (
@@ -147,8 +147,7 @@ def reorder(x, order, reverse=False):  # x is NxTxD1x...xDk, order is NxT'
     u = x.reshape(x.size()[:2] + (-1,))
     order = order.unsqueeze(-1).expand(-1, -1, u.size(-1))
     if reverse:
-        v = u.new(u.size())
-        v.scatter_(1, order, u)
+        v = u.new(u.size()).scatter_(1, order, u)
     else:
         v = u.gather(1, order)
     v = v.reshape(v.size()[:2] + x.size()[2:])
@@ -160,6 +159,12 @@ def shuffle(x, fixed_len):
     return reorder(x, order), order
 
 
+def eval_mygpt(model, input, mode="standard", fixed_len=0):
+    x, order = shuffle(input, fixed_len)
+    x = model(mygpt.BracketedSequence(x), mode=mode, order=order).x
+    return reorder(x, order, reverse=True)
+
+
 ######################################################################
 
 # ar_mask is a Boolean matrix of same shape as input, with 1s on the
@@ -197,9 +202,7 @@ def compute_perplexity(model, task, fixed_len, split="train"):
 
         for input in task.batches(split=split):
             input = input.to(device)
-            x, order = shuffle(input, fixed_len)
-            x = model(mygpt.BracketedSequence(x), order=order).x
-            output = reorder(x, order, reverse=True)
+            output = eval_mygpt(model, input, fixed_len=fixed_len)
             loss = F.cross_entropy(output.transpose(1, 2), input)
             acc_loss += loss.item() * input.size(0)
             nb_samples += input.size(0)
@@ -263,9 +266,9 @@ def oneshot(gpt, task):
 
         acc_train_loss, nb_train_samples = 0, 0
         for mazes, policies in task.policy_batches(split="train"):
-            x, order = shuffle(mazes, task.height * task.width)
-            x = gpt(mygpt.BracketedSequence(x), mode=args.oneshot_input, order=order).x
-            output_gpt = reorder(x, order, reverse=True)
+            output_gpt = eval_mygpt(
+                gpt, mazes, mode=args.oneshot_input, fixed_len=task.height * task.width
+            )
             output = model(output_gpt)
 
             loss = compute_loss(mazes, output, policies, task.height, task.width)
@@ -278,9 +281,9 @@ def oneshot(gpt, task):
 
         acc_test_loss, nb_test_samples = 0, 0
         for mazes, policies in task.policy_batches(split="test"):
-            x, order = shuffle(mazes, task.height * task.width)
-            x = gpt(mygpt.BracketedSequence(x), mode=args.oneshot_input, order=order).x
-            output_gpt = reorder(x, order, reverse=True)
+            output_gpt = eval_mygpt(
+                gpt, mazes, mode=args.oneshot_input, fixed_len=task.height * task.width
+            )
             output = model(output_gpt)
             loss = compute_loss(mazes, output, policies, task.height, task.width)
             acc_test_loss += loss.item() * mazes.size(0)
@@ -293,9 +296,9 @@ def oneshot(gpt, task):
         # -------------------
         mazes = task.test_input[:32, : task.height * task.width]
         policies = task.test_policies[:32]
-        x, order = shuffle(mazes, task.height * task.width)
-        x = gpt(mygpt.BracketedSequence(x), mode=args.oneshot_input, order=order).x
-        output_gpt = reorder(x, order, reverse=True)
+        output_gpt = eval_mygpt(
+            gpt, mazes, mode=args.oneshot_input, fixed_len=task.height * task.width
+        )
         output = model(output_gpt)
         if args.oneshot_output == "policy":
             targets = policies.permute(0, 2, 1)
@@ -628,9 +631,9 @@ for n_epoch in range(nb_epochs_finished, args.nb_epochs):
 
     for input in task.batches(split="train"):
         input = input.to(device)
-        x, order = shuffle(input, task.height * task.width)
-        x = model(mygpt.BracketedSequence(x), order=order).x
-        output = reorder(x, order, reverse=True)
+        output = eval_mygpt(
+            model, input, mode=args.oneshot_input, fixed_len=task.height * task.width
+        )
         loss = F.cross_entropy(output.transpose(1, 2), input)
         acc_train_loss += loss.item() * input.size(0)
         nb_train_samples += input.size(0)