X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=mygpt.py;h=c061eb4c84a39e5cb58b0e9ce093812017c07c72;hb=03100792df9e52b739bbe4692bed6c4f6b575242;hp=7105e97c351138b6dbb9e382d99c29293b1f594c;hpb=b458d5aa1f2ba736807e87b65ccad2f96b216a10;p=mygptrnn.git

diff --git a/mygpt.py b/mygpt.py
index 7105e97..c061eb4 100755
--- a/mygpt.py
+++ b/mygpt.py
@@ -442,7 +442,8 @@ class KVRec(nn.Module):
 
 
 # Returns a tensor with an additional index at rank win_dim, that move
-# along the same dimension as dim, on a domain {0...win_size-1}
+# along the same dimension as dim, on a domain {0...win_size-1}, and
+# dim is restricted on a domain reduced by win_size-1 values.
 
 
 def moving_window(x, dim, win_dim, win_size):
@@ -544,6 +545,8 @@ class Caterpillar(nn.Module):
             torch.einsum("ntc,hec->nhet", X, self.w_G) + self.b_G[None, :, :, None]
         ).sigmoid()
 
+        G = F.dropout(G, self.attention_dropout, self.training)
+
         V = torch.einsum("ntc,hdc->nhtd", X, self.w_V)
         K = torch.einsum("ntc,hdc->nhtd", X, self.w_K)