From: Francois Fleuret Date: Fri, 29 Jul 2022 03:45:46 +0000 (+0200) Subject: Update. X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=52c6bd98650c846459f10e8303dd2e6c7ba2a68f;p=mygpt.git Update. --- diff --git a/main.py b/main.py index 427a83a..83227bb 100755 --- a/main.py +++ b/main.py @@ -165,6 +165,12 @@ class TaskPicoCLVR(Task): id_descr = [ [ self.token2id[u] for u in s ] for s in token_descr ] return torch.tensor(id_descr, device = self.device) + def trim(self, x, token = ''): + n = self.token2id[token] + i = (1 - (F.pad(x, (1, 1), value = n) == n).min(0).values.long()).cumsum(0) + a, b = (i == 0).nonzero().max(), (i == i.max()).nonzero().min() + return x[:, a:b] + def __init__(self, batch_size, height, width, nb_colors = 5, device = torch.device('cpu')): @@ -201,13 +207,13 @@ class TaskPicoCLVR(Task): assert split in { 'train', 'test' } input = self.train_input if split == 'train' else self.test_input for batch in tqdm.tqdm(input.split(self.batch_size), desc = f'epoch-{split}'): - yield batch + yield self.trim(batch) def vocabulary_size(self): return len(self.token2id) def produce_results(self, n_epoch, model): - nb_tokens = self.height * self.width + 3 + nb_tokens_to_generate = self.height * self.width + 3 result_descr = [ ] nb_per_primer = 8 @@ -218,15 +224,26 @@ class TaskPicoCLVR(Task): 'green bottom yellow bottom green left of blue yellow right of blue blue top ', ]: - for k in range(nb_per_primer): - results = autoregression( - model, self.batch_size, - nb_samples = 1, nb_tokens_to_generate = nb_tokens, - primer = self.tensorize([ primer_descr ]), - device = self.device - ) - r = ' '.join([ self.id2token[t.item()] for t in results.flatten() ]) - result_descr.append(r) + results = autoregression( + model, + self.batch_size, + nb_samples = nb_per_primer, + nb_tokens_to_generate = nb_tokens_to_generate, + primer = self.tensorize([ primer_descr ]).expand(nb_per_primer, -1), + device = self.device + ) + + l = [ ' '.join([ self.id2token[t.item()] for t in r ]) for r in results ] + result_descr += l + + np = picoclvr.nb_properties( + result_descr, + height = self.height, width = self.width + ) + + nb_requested_properties, _, nb_missing_properties = zip(*np) + + log_string(f'nb_requested_properties {sum(nb_requested_properties) / len(result_descr):.02f} nb_missing_properties {sum(nb_missing_properties) / len(result_descr):.02f}') img = [ picoclvr.descr2img(d, height = self.height, width = self.width) @@ -241,15 +258,6 @@ class TaskPicoCLVR(Task): ) log_string(f'wrote {image_name}') - np = picoclvr.nb_properties( - result_descr, - height = self.height, width = self.width - ) - - nb_requested_properties, _, nb_missing_properties = zip(*np) - - log_string(f'nb_requested_properties {sum(nb_requested_properties) / len(result_descr):.02f} nb_missing_properties {sum(nb_missing_properties) / len(result_descr):.02f}') - ###################################################################### class TaskWiki103(Task): @@ -462,7 +470,6 @@ for input in task.batches(split = 'train'): token_probas = token_count / token_count.sum() entropy = -torch.xlogy(token_probas, token_probas).sum() train_set_perplexity = math.exp(entropy) -#log_string(f'train set perplexity {train_set_perplexity}') for k in range(nb_epochs_finished, nb_epochs): diff --git a/mygpt.py b/mygpt.py index 212e1a5..d6879dc 100755 --- a/mygpt.py +++ b/mygpt.py @@ -104,7 +104,7 @@ class MyGPT(nn.Module): for _ in range(nb_blocks): trunk_blocks += [ Residual( - nn.LayerNorm(dim_model), + nn.LayerNorm((dim_model,)), QKVAttention( dim_in = dim_model, dim_qk = dim_keys, @@ -114,7 +114,7 @@ class MyGPT(nn.Module): ), ), Residual( - nn.LayerNorm(dim_model), + nn.LayerNorm((dim_model,)), nn.Linear(in_features = dim_model, out_features = dim_hidden), nn.ReLU(), nn.Linear(in_features = dim_hidden, out_features = dim_model), @@ -131,7 +131,8 @@ class MyGPT(nn.Module): x = self.embedding(x) x = self.trunk(x) x = self.readout(x) - return x[:, :-1] + x = F.pad(x, (0, 0, 0, -1)) + return x ######################################################################