From: François Fleuret <francois@fleuret.org>
Date: Thu, 23 Mar 2023 08:24:59 +0000 (+0100)
Subject: Update
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=706569c57bfbdc7e9a0791cdb608236208012710;p=beaver.git

Update
---

diff --git a/beaver.py b/beaver.py
index 8fe9a9b..69116ea 100755
--- a/beaver.py
+++ b/beaver.py
@@ -135,24 +135,33 @@ def generation_order(x, fixed_len):
     if args.random_regression_order:
         order = torch.rand(x.size(), device=x.device)
         order[:, :fixed_len] = torch.linspace(-2, -1, fixed_len, device=order.device)
-        return order.sort(1).indices
+        order = order.sort(1).indices
     else:
-        return (
+        order = (
             torch.arange(x.size(1), device=x.device).unsqueeze(0).expand(x.size(0), -1)
         )
+    return order
 
 
-def shuffle(x, order, reorder=False):
-    if x.dim() == 3:
-        order = order.unsqueeze(-1).expand(-1, -1, x.size(-1))
-    if reorder:
-        y = x.new(x.size())
-        y.scatter_(1, order, x)
-        return y
+def reorder(x, order, back=False):  # x is NxTxD1x...xDk, order is NxT'
+    u = x.reshape(x.size()[:2] + (-1,))
+    order = order.unsqueeze(-1).expand(-1, -1, u.size(-1))
+    if back:
+        v = u.new(u.size())
+        v.scatter_(1, order, u)
     else:
-        return x.gather(1, order)
+        v = u.gather(1, order)
+    v = v.reshape(v.size()[:2] + x.size()[2:])
+    return v
 
 
+def shuffle(x, fixed_len):
+    order = generation_order(x, fixed_len)
+    return reorder(x, order), order
+
+
+######################################################################
+
 # ar_mask is a Boolean matrix of same shape as input, with 1s on the
 # tokens that should be generated
 
@@ -186,8 +195,7 @@ def compute_perplexity(model, split="train"):
 
         for input in task.batches(split=split):
             input = input.to(device)
-            order = generation_order(input, task.height * task.width)
-            input = shuffle(input, order)
+            input, order = shuffle(input, task.height * task.width)
             output = model(mygpt.BracketedSequence(input), order=order).x
             loss = F.cross_entropy(output.transpose(1, 2), input)
             acc_loss += loss.item() * input.size(0)
@@ -252,10 +260,9 @@ def oneshot(gpt, task):
 
         acc_train_loss, nb_train_samples = 0, 0
         for mazes, policies in task.policy_batches(split="train"):
-            order = generation_order(mazes, task.height * task.width)
-            x = shuffle(mazes, order)
+            x, order = shuffle(mazes, task.height * task.width)
             x = gpt(mygpt.BracketedSequence(x), mode=args.oneshot_input, order=order).x
-            output_gpt = shuffle(x, order, reorder=True)
+            output_gpt = reorder(x, order, back=True)
             output = model(output_gpt)
 
             loss = compute_loss(mazes, output, policies, task.height, task.width)
@@ -268,10 +275,9 @@ def oneshot(gpt, task):
 
         acc_test_loss, nb_test_samples = 0, 0
         for mazes, policies in task.policy_batches(split="test"):
-            order = generation_order(mazes, task.height * task.width)
-            x = shuffle(mazes, order)
+            x, order = shuffle(mazes, task.height * task.width)
             x = gpt(mygpt.BracketedSequence(x), mode=args.oneshot_input, order=order).x
-            output_gpt = shuffle(x, order, reorder=True)
+            output_gpt = reorder(x, order, back=True)
             output = model(output_gpt)
             loss = compute_loss(mazes, output, policies, task.height, task.width)
             acc_test_loss += loss.item() * mazes.size(0)
@@ -284,10 +290,9 @@ def oneshot(gpt, task):
         # -------------------
         mazes = task.test_input[:32, : task.height * task.width]
         policies = task.test_policies[:32]
-        order = generation_order(mazes, task.height * task.width)
-        x = shuffle(mazes, order)
+        x, order = shuffle(mazes, task.height * task.width)
         x = gpt(mygpt.BracketedSequence(x), mode=args.oneshot_input, order=order).x
-        output_gpt = shuffle(x, order, reorder=True)
+        output_gpt = reorder(x, order, back=True)
         output = model(output_gpt)
         if args.oneshot_output == "policy":
             targets = policies.permute(0, 2, 1)
@@ -426,11 +431,11 @@ class TaskMaze(Task):
             ar_mask = result.new_zeros(result.size())
             ar_mask[:, self.height * self.width :] = 1
             result *= 1 - ar_mask
-            order = generation_order(result, self.height * self.width)
+            result, order = shuffle(result, self.height * self.width)
             masked_inplace_autoregression(
                 model, self.batch_size, result, ar_mask, order=order
             )
-            result = shuffle(result, order, reorder=True)
+            result = reorder(result, order, back=True)
             mazes, paths = self.seq2map(result)
             nb_correct += maze.path_correctness(mazes, paths).long().sum()
             nb_total += mazes.size(0)
@@ -608,8 +613,7 @@ for n_epoch in range(nb_epochs_finished, args.nb_epochs):
 
     for input in task.batches(split="train"):
         input = input.to(device)
-        order = generation_order(input, task.height * task.width)
-        input = shuffle(input, order)
+        input, order = shuffle(input, task.height * task.width)
         output = model(mygpt.BracketedSequence(input), order=order).x
         loss = F.cross_entropy(output.transpose(1, 2), input)
         acc_train_loss += loss.item() * input.size(0)