X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=mygpt.py;h=0b63ac800ccccc96f014c0b5c149476814698820;hb=280f38363e34e202d38e6f7c00288329ab067a81;hp=5ea4668203f0f4eaf9ccbc586f2d58a348fe0a3f;hpb=3602bfe2c4e1cd513759bf45cb83f8c2d914674b;p=beaver.git

diff --git a/mygpt.py b/mygpt.py
index 5ea4668..0b63ac8 100755
--- a/mygpt.py
+++ b/mygpt.py
@@ -14,19 +14,6 @@ from torch.nn import functional as F
 
 ######################################################################
 
-
-class WithResidual(nn.Module):
-    def __init__(self, *f):
-        super().__init__()
-        self.f = f[0] if len(f) == 1 else nn.Sequential(*f)
-
-    def forward(self, bs):
-        bs.x = bs.x + self.f(bs).x
-        return bs
-
-
-######################################################################
-
 # A BracketedSequence is a BxTx... tensor with a first and a nb time
 # steps to compute.
 
@@ -57,6 +44,19 @@ class BracketedSequence:
 ######################################################################
 
 
+class WithResidual(nn.Module):
+    def __init__(self, *f):
+        super().__init__()
+        self.f = f[0] if len(f) == 1 else nn.Sequential(*f)
+
+    def forward(self, bs):
+        bs.x = bs.x + self.f(bs).x
+        return bs
+
+
+######################################################################
+
+
 class CacheWrapper(nn.Module):
     def __init__(self, *f):
         super().__init__()
@@ -197,7 +197,6 @@ class MyGPT(nn.Module):
         dropout=0.0,
         len_max=1e5,
     ):
-
         super().__init__()
 
         assert dim_model % nb_heads == 0
@@ -247,18 +246,28 @@ class MyGPT(nn.Module):
                     m.bias.zero_()
                     m.weight.fill_(1.0)
 
-    def forward(self, bs):
+    def forward(self, bs, mode="standard"):
         bs.x = F.pad(bs.x, (1, -1))
         bs = self.embedding(bs)
-        bs = self.trunk(bs)
-        bs = self.readout(bs)
+        if mode == "standard":
+            bs = self.trunk(bs)
+            bs = self.readout(bs)
+        elif mode == "head":
+            bs = self.trunk(bs)
+        elif mode == "deep":
+            r = []
+            for l in self.trunk:
+                bs = l(bs)
+                r += [bs.slice()]
+            bs = BracketedSequence(torch.cat(r, -1))
+        else:
+            raise ValueError
         return bs
 
 
 ######################################################################
 
 if __name__ == "__main__":
-
     print("Basic check.")
 
     vocabulary_size = 10