5 import torch, torchvision
8 from torch.nn import functional as F
13 ######################################################################
16 def save_attention_image(
20 # An iterable set of BxHxTxT attention matrices
29 # do not draw links with a lesser attention
31 # draw only the strongest links necessary to reache
33 min_total_attention=None,
34 # draw only the top k links
38 attention = torch.cat(
39 [x[n_sample : n_sample + 1, n_head] for x in attention_arrays], dim=0
43 attention = attention * (
44 attention.sort(dim=-1, descending=True).indices < k_top
47 if min_total_attention is not None:
48 s = attention.sort(dim=-1)
49 m = 1 - (s.values.cumsum(-1) < 1 - min_total_attention).long()
50 b = m.new(attention.size()).scatter_(dim=-1, index=s.indices, src=m)
51 attention = attention * b
53 surface = cairo.RecordingSurface(cairo.CONTENT_COLOR_ALPHA, None)
55 ctx = cairo.Context(surface)
56 ctx.scale(pixel_scale, pixel_scale)
58 ctx.set_source_rgb(0.0, 0.0, 0.0)
59 ctx.set_font_size(4.0)
60 # ctx.select_font_face("Arial", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
64 for d in range(attention.size(0)):
66 ni = torch.arange(at.size(0))[:, None].expand_as(at)
67 nj = torch.arange(at.size(1))[None, :].expand_as(at)
73 for i, j, a in zip(ni, nj, at):
74 if a > 0 and a >= min_link_attention:
76 ctx.set_source_rgb(c, c, c)
77 ctx.set_line_width(0.5)
78 ax, ay = j * token_gap, y - y_eps
80 dx, dy = i * token_gap, y - layer_gap + y_eps
82 bx, by = ax, ay - layer_gap * 0.5
83 cx, cy = dx, dy + layer_gap * 0.5
84 ctx.curve_to(bx, by, cx, cy, dx, dy)
90 for d in range(0, attention.size(0) + 1):
91 for n in range(attention.size(-1)):
92 xc, yc = n * token_gap, -d * layer_gap
93 ctx.set_source_rgb(1.0, 1.0, 1.0)
94 ctx.arc(xc, yc, token_gap / 10, 0, 2 * math.pi)
96 ctx.set_source_rgb(0.0, 0.0, 0.0)
97 ctx.arc(xc, yc, token_gap / 20, 0, 2 * math.pi)
100 ctx.set_source_rgb(0.0, 0.0, 0.0)
102 for k, t in enumerate(tokens_input):
111 ) = ctx.text_extents(s)
112 ctx.move_to(k * token_gap - width_t / 2, token_gap / 5 - y_bearing)
115 for k, t in enumerate(tokens_output):
124 ) = ctx.text_extents(s)
126 k * token_gap - width_t / 2, -token_gap / 5 - attention.size(0) * layer_gap
130 x, y, width, height = surface.ink_extents()
134 height += 2 * padding
135 pdf_surface = cairo.PDFSurface(filename, width, height)
136 ctx_pdf = cairo.Context(pdf_surface)
137 ctx_pdf.set_source_surface(surface, -x, -y)
142 ######################################################################
144 if __name__ == "__main__":
147 tokens_output = ["<wat>", 2, 3, 4, "<end>"]
148 tokens_input = [""] + tokens_output[:-1]
151 x = torch.randint(vocabulary_size, (1, len(tokens_input)))
154 vocabulary_size=vocabulary_size,
165 model.record_attention()
167 y1 = model(mygpt.BracketedSequence(x)).x
169 attention = model.retrieve_attention()
171 save_attention_image(
177 min_total_attention=0.9,