X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=graph.py;h=07e376a3af86d24874523775e26f16132a489512;hb=21ed4aa91d0f1ac87ec684d8808e5ced552ad457;hp=c286388d6298eba52f7ad88ed9bffa0d5ab2af9a;hpb=8492656cf0cc5de4f7e2c4aa8ccb717193293b40;p=picoclvr.git

diff --git a/graph.py b/graph.py
index c286388..07e376a 100755
--- a/graph.py
+++ b/graph.py
@@ -14,24 +14,26 @@ import cairo
 
 
 def save_attention_image(
+    # image to save
     filename,
     tokens_input,
     tokens_output,
-    # An iterable set of BxHxTxT attention matrices
+    # list of 2d tensors T2xT1, T3xT2, ..., TkxTk-1
     attention_matrices,
-    pixel_scale=8,
-    token_gap=15,
-    layer_gap=25,
-    y_eps=0.5,
-    padding=10,
     # do not draw links with a lesser attention
     min_link_attention=0,
-    # draw only the strongest links necessary to reache
-    # min_total_attention
+    # draw only the strongest links necessary so that their summed
+    # attention is above min_total_attention
     min_total_attention=None,
     # draw only the top k links
     k_top=None,
+    # the purely graphical settings
     curved=True,
+    pixel_scale=8,
+    token_gap=15,
+    layer_gap=25,
+    y_eps=0.5,
+    padding=10,
 ):
     if k_top is not None:
         am = []
@@ -60,7 +62,7 @@ def save_attention_image(
 
     ctx.set_line_width(0.25)
     for d in range(len(attention_matrices)):
-        at = attention_matrices[d]
+        at = attention_matrices[d].to("cpu")
         ni = torch.arange(at.size(0))[:, None].expand_as(at)
         nj = torch.arange(at.size(1))[None, :].expand_as(at)
         at = at.flatten()
@@ -111,7 +113,7 @@ def save_attention_image(
             x_advance,
             y_advance,
         ) = ctx.text_extents(s)
-        ctx.move_to(k * token_gap - width_t / 2, token_gap / 5 - y_bearing)
+        ctx.move_to(k * token_gap - width_t / 2, 2 * token_gap / 5)
         ctx.show_text(s)
 
     for k, t in enumerate(tokens_output):
@@ -147,7 +149,7 @@ def save_attention_image(
 if __name__ == "__main__":
     import mygpt
 
-    tokens_output = ["<wat>", 2, 3, 4, "<end>"]
+    tokens_output = ["<wat>", "-", 3, 4, "<end>"]
     tokens_input = [""] + tokens_output[:-1]
 
     vocabulary_size = 3
@@ -161,7 +163,7 @@ if __name__ == "__main__":
         nb_heads=2,
         nb_blocks=5,
         dropout=0.1,
-        #causal=True,
+        causal=True,
     )
 
     model.eval()
@@ -171,10 +173,7 @@ if __name__ == "__main__":
 
     attention_matrices = [m[0, 0] for m in model.retrieve_attention()]
 
-    
-
-    # attention_matrices = [ torch.rand(3,5), torch.rand(8,3), torch.rand(5,8) ]
-    # for a in attention_matrices: a=a/a.sum(-1,keepdim=True)
+    # attention_matrices = [torch.rand(*s) for s in [ (4,5),(3,4),(8,3),(5,8) ]]
 
     save_attention_image(
         "attention.pdf",