projects
/
culture.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Update.
[culture.git]
/
tasks.py
diff --git
a/tasks.py
b/tasks.py
index
0eed2aa
..
0c92af9
100755
(executable)
--- a/
tasks.py
+++ b/
tasks.py
@@
-1284,7
+1284,7
@@
class RPL(Task):
)
if save_attention_image is not None:
)
if save_attention_image is not None:
- input = self.test_input[:1
0
]
+ input = self.test_input[:1]
result = input.clone()
s = (result == self.t_prog).long()
ar_mask = (s.cumsum(dim=1) - s).clamp(min=0, max=1)
result = input.clone()
s = (result == self.t_prog).long()
ar_mask = (s.cumsum(dim=1) - s).clamp(min=0, max=1)
@@
-1305,24
+1305,23
@@
class RPL(Task):
model.record_attention(True)
model(BracketedSequence(result))
model.train(t)
model.record_attention(True)
model(BracketedSequence(result))
model.train(t)
-
attention
= model.retrieve_attention()
+
ram
= model.retrieve_attention()
model.record_attention(False)
model.record_attention(False)
- n_sample = 0
- tokens_output = [self.id2token[i.item()] for i in result[n_sample]]
+ tokens_output = [self.id2token[i.item()] for i in result[0]]
tokens_input = ["n/a"] + tokens_output[:-1]
tokens_input = ["n/a"] + tokens_output[:-1]
- for n_head in range(
attention
[0].size(1)):
+ for n_head in range(
ram
[0].size(1)):
filename = f"rpl_attention_{n_epoch}_h{n_head}.pdf"
filename = f"rpl_attention_{n_epoch}_h{n_head}.pdf"
+ attention_matrices = [m[0, n_head] for m in ram]
save_attention_image(
filename,
tokens_input,
tokens_output,
save_attention_image(
filename,
tokens_input,
tokens_output,
- attention,
- n_sample=n_sample,
- n_head=n_head,
+ attention_matrices,
token_gap=12,
token_gap=12,
- layer_gap=40,
- # k_top=2,
+ layer_gap=50,
+ k_top=10,
+ # min_total_attention=0.9,
)
logger(f"wrote {filename}")
)
logger(f"wrote {filename}")