From: François Fleuret <francois@fleuret.org>
Date: Sat, 22 Jul 2023 21:13:29 +0000 (+0200)
Subject: Update.
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=95717a8bf88159051f9c4b8862b0b643187826e9;p=picoclvr.git

Update.
---

diff --git a/graph.py b/graph.py
index c286388..08f1170 100755
--- a/graph.py
+++ b/graph.py
@@ -14,16 +14,10 @@ import cairo
 
 
 def save_attention_image(
-    filename,
+    filename,  # image to save
     tokens_input,
     tokens_output,
-    # An iterable set of BxHxTxT attention matrices
-    attention_matrices,
-    pixel_scale=8,
-    token_gap=15,
-    layer_gap=25,
-    y_eps=0.5,
-    padding=10,
+    attention_matrices,  # list of 2d tensors T1xT2, T2xT3, ..., Tk-1xTk
     # do not draw links with a lesser attention
     min_link_attention=0,
     # draw only the strongest links necessary to reache
@@ -32,6 +26,11 @@ def save_attention_image(
     # draw only the top k links
     k_top=None,
     curved=True,
+    pixel_scale=8,
+    token_gap=15,
+    layer_gap=25,
+    y_eps=0.5,
+    padding=10,
 ):
     if k_top is not None:
         am = []
@@ -161,7 +160,7 @@ if __name__ == "__main__":
         nb_heads=2,
         nb_blocks=5,
         dropout=0.1,
-        #causal=True,
+        causal=True,
     )
 
     model.eval()
@@ -171,8 +170,6 @@ if __name__ == "__main__":
 
     attention_matrices = [m[0, 0] for m in model.retrieve_attention()]
 
-    
-
     # attention_matrices = [ torch.rand(3,5), torch.rand(8,3), torch.rand(5,8) ]
     # for a in attention_matrices: a=a/a.sum(-1,keepdim=True)
 
diff --git a/tasks.py b/tasks.py
index 0c92af9..a27b836 100755
--- a/tasks.py
+++ b/tasks.py
@@ -140,7 +140,6 @@ class ProblemLevel2(Problem):
             num_classes=self.len_source,
         )
         source1 = torch.rand(nb, 10).sort(dim=1).indices[:, : self.len_source]
-        # source1 = torch.randint(10, (nb, self.len_source))
         marker1 = torch.full((nb, 1), 10)
         result1 = operators.bmm(source1[:, :, None]).squeeze(-1)
         marker2 = torch.full((nb, 1), 11)
@@ -1311,17 +1310,19 @@ class RPL(Task):
             tokens_output = [self.id2token[i.item()] for i in result[0]]
             tokens_input = ["n/a"] + tokens_output[:-1]
             for n_head in range(ram[0].size(1)):
-                filename = f"rpl_attention_{n_epoch}_h{n_head}.pdf"
+                filename = os.path.join(
+                    result_dir, f"rpl_attention_{n_epoch}_h{n_head}.pdf"
+                )
                 attention_matrices = [m[0, n_head] for m in ram]
                 save_attention_image(
                     filename,
                     tokens_input,
                     tokens_output,
                     attention_matrices,
-                    token_gap=12,
-                    layer_gap=50,
                     k_top=10,
                     # min_total_attention=0.9,
+                    token_gap=12,
+                    layer_gap=50,
                 )
                 logger(f"wrote {filename}")