- t = [ [ self.token2id[u] for u in s ] for s in descr ]
- return torch.tensor(t, device = self.device)
+ token_descr = [ s.strip().split(' ') for s in descr ]
+ l = max([ len(s) for s in token_descr ])
+ #token_descr = [ [ '<nul>' ] * (l - len(s)) + s for s in token_descr ]
+ token_descr = [ s + [ '<nul>' ] * (l - len(s)) for s in token_descr ]
+ id_descr = [ [ self.token2id[u] for u in s ] for s in token_descr ]
+ return torch.tensor(id_descr, device = self.device)
+
+ def trim(self, x, token = '<nul>'):
+ n = self.token2id[token]
+ i = (1 - (F.pad(x, (1, 1), value = n) == n).min(0).values.long()).cumsum(0)
+ a, b = (i == 0).nonzero().max(), (i == i.max()).nonzero().min()
+ return x[:, a:b]