def tensorize(self, descr):
token_descr = [ s.strip().split(' ') for s in descr ]
l = max([ len(s) for s in token_descr ])
- #token_descr = [ [ '<nul>' ] * (l - len(s)) + s for s in token_descr ]
- token_descr = [ s + [ '<nul>' ] * (l - len(s)) for s in token_descr ]
+ token_descr = [ [ '<nul>' ] * (l - len(s)) + s for s in token_descr ]
+ #token_descr = [ s + [ '<nul>' ] * (l - len(s)) for s in token_descr ]
id_descr = [ [ self.token2id[u] for u in s ] for s in token_descr ]
return torch.tensor(id_descr, device = self.device)