X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=main.py;h=5f3e8cffe5bce08167141cb79e116e1f5a9e336a;hb=1ad9ea3cca4489b07bad8521966382f66a493eea;hp=1cd7342184b02fa904c60c75f961d93664cc95e8;hpb=0553fc413c4af68bc777b70a6236c622a3b5242f;p=mygpt.git diff --git a/main.py b/main.py index 1cd7342..5f3e8cf 100755 --- a/main.py +++ b/main.py @@ -18,15 +18,11 @@ import mygpt device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') ###################################################################### - parser = argparse.ArgumentParser(description = 'My own GPT.') parser.add_argument('--log_filename', type = str, default = 'train.log') -parser.add_argument('--download', - action='store_true', default = False) - parser.add_argument('--seed', type = int, default = 0) @@ -114,8 +110,8 @@ for n in vars(args): ###################################################################### def autoregression( - model, - nb_samples, nb_tokens_to_generate, starting_input = None, + model, batch_size, + nb_samples, nb_tokens_to_generate, primer = None, device = torch.device('cpu') ): results = torch.zeros( @@ -123,13 +119,13 @@ def autoregression( dtype = torch.int64, device = device ) - if starting_input is None: + if primer is None: first = 0 else: - first = starting_input.size(1) - results = torch.cat((starting_input, results), 1) + first = primer.size(1) + results = torch.cat((primer, results), 1) - for input in results.split(args.batch_size): + for input in results.split(batch_size): for s in tqdm.tqdm(range(first, input.size(1)), desc = 'synth'): output = model(input) logits = output[:, s] @@ -151,7 +147,7 @@ class Task: def vocabulary_size(self): pass - def produce_results(self, n_epoch, model, nb_tokens = 50): + def produce_results(self, n_epoch, model): pass ###################################################################### @@ -160,6 +156,10 @@ import picoclvr class TaskPicoCLVR(Task): + def tensorize(self, descr): + t = [ [ self.token2id[u] for u in s ] for s in descr ] + return torch.tensor(t, device = self.device) + def __init__(self, batch_size, height, width, nb_colors = 5, device = torch.device('cpu')): @@ -173,7 +173,8 @@ class TaskPicoCLVR(Task): descr = [ s.strip().split(' ') for s in descr ] l = max([ len(s) for s in descr ]) - descr = [ s + [ '' ] * (l - len(s)) for s in descr ] + #descr = [ [ '' ] * (l - len(s)) + s for s in descr ] + descr = [ s + [ '' ] * (l - len(s)) for s in descr ] return descr @@ -194,49 +195,25 @@ class TaskPicoCLVR(Task): self.token2id = dict([ (t, n) for n, t in enumerate(tokens) ]) self.id2token = dict([ (n, t) for n, t in enumerate(tokens) ]) - t = [ [ self.token2id[u] for u in s ] for s in self.train_descr ] - self.train_input = torch.tensor(t, device = self.device) - t = [ [ self.token2id[u] for u in s ] for s in self.test_descr ] - self.test_input = torch.tensor(t, device = self.device) + # Tokenize the train and test sets + self.train_input = self.tensorize(self.train_descr) + self.test_input = self.tensorize(self.test_descr) def batches(self, split = 'train'): assert split in { 'train', 'test' } - if split == 'train': - for batch in tqdm.tqdm(self.train_input.split(self.batch_size), desc = f'epoch-{split}'): - yield batch - else: - for batch in tqdm.tqdm(self.test_input.split(self.batch_size), desc = f'epoch-{split}'): - yield batch + input = self.train_input if split == 'train' else self.test_input + for batch in tqdm.tqdm(input.split(self.batch_size), desc = f'epoch-{split}'): + yield batch def vocabulary_size(self): return len(self.token2id) - def generate(self, primer, model, nb_tokens): - t_primer = primer.strip().split(' ') - t_generated = [ ] - - for j in range(nb_tokens): - t = [ [ self.token2id[u] for u in t_primer + t_generated ] ] - input = torch.tensor(t, device = self.device) - input = F.pad(input, (0, 1)) # Add the next token, the one to predict - output = model(input) - logits = output[0, -1] - if args.synthesis_sampling: - dist = torch.distributions.categorical.Categorical(logits = logits) - t_next = dist.sample() - else: - t_next = logits.argmax() - t_generated.append(self.id2token[t_next.item()]) - - return ' '.join(t_primer + t_generated) - - def produce_results(self, n_epoch, model, nb_tokens = None): - if nb_tokens is None: - nb_tokens = self.height * self.width + 3 - descr = [ ] + def produce_results(self, n_epoch, model): + nb_tokens = self.height * self.width + 3 + result_descr = [ ] nb_per_primer = 8 - for primer in [ + for primer_descr in [ 'red above green green top blue right of red ', 'there is red there is yellow there is blue ', 'red below yellow yellow below green green below blue red right yellow left green right blue left ', @@ -244,9 +221,20 @@ class TaskPicoCLVR(Task): ]: for k in range(nb_per_primer): - descr.append(self.generate(primer, model, nb_tokens)) + results = autoregression( + model, self.batch_size, + nb_samples = 1, nb_tokens = nb_tokens, + primer = self.tensorize(primer_descr), + device = self.device + ) + r = ' '.join([ self.id2token[t.item()] for t in results.flatten() ]) + result_descr.append(r) + + img = [ + picoclvr.descr2img(d, height = self.height, width = self.width) + for d in result_descr + ] - img = [ picoclvr.descr2img(d, height = self.height, width = self.width) for d in descr ] img = torch.cat(img, 0) image_name = f'result_picoclvr_{n_epoch:04d}.png' torchvision.utils.save_image( @@ -255,14 +243,14 @@ class TaskPicoCLVR(Task): ) log_string(f'wrote {image_name}') - nb_missing = sum( [ - x[2] for x in picoclvr.nb_missing_properties( - descr, - height = self.height, width = self.width - ) - ] ) + np = picoclvr.nb_properties( + result_descr, + height = self.height, width = self.width + ) + + nb_requested_properties, _, nb_missing_properties = zip(*np) - log_string(f'nb_missing {nb_missing / len(descr):.02f}') + log_string(f'nb_requested_properties {sum(nb_requested_properties) / len(result_descr):.02f} nb_missing_properties {sum(nb_missing_properties) / len(result_descr):.02f}') ###################################################################### @@ -290,7 +278,7 @@ class TaskWiki103(Task): self.vocab = torchtext.vocab.build_vocab_from_iterator( yield_tokens(), - specials = [ '', '' ], + specials = [ '', '' ], min_freq = self.min_freq ) @@ -298,7 +286,7 @@ class TaskWiki103(Task): def tensorize(self, s): a = max(len(x) for x in s) - return torch.tensor([ self.vocab(x + [ '' ] * (a - len(x))) for x in s ]) + return torch.tensor([ self.vocab(x + [ '' ] * (a - len(x))) for x in s ]) def yield_batches(self, ds): s = [ ] @@ -325,7 +313,8 @@ class TaskWiki103(Task): def vocabulary_size(self): return len(self.vocab) - def produce_results(self, n_epoch, model, nb_tokens = 50): + def produce_results(self, n_epoch, model): + nb_tokens = 50 file_name = f'result_wiki103_{n_epoch:04d}.txt' with open(file_name, 'w') as outfile: @@ -355,7 +344,7 @@ class TaskWiki103(Task): else: t_next = logits.argmax() t_generated.append(self.vocab.lookup_token(t_next)) - if t_generated[-1] == '': break + if t_generated[-1] == '': break s = ' '.join(t_generated) @@ -386,8 +375,9 @@ class TaskMNIST(Task): def vocabulary_size(self): return 256 - def produce_results(self, n_epoch, model, nb_samples = 64): - results = autoregression(model, nb_samples, 28 * 28, device = self.device) + def produce_results(self, n_epoch, model): + nb_samples = 64 + results = autoregression(model, self.batch_size, nb_samples, 28 * 28, device = self.device) image_name = f'result_mnist_{n_epoch:04d}.png' torchvision.utils.save_image(1 - results.reshape(-1, 1, 28, 28) / 255., image_name, nrow = 16, pad_value = 0.8) @@ -446,7 +436,7 @@ else: nb_epochs_finished = 0 if args.no_checkpoint: - log_string(f'Not trying to load checkpoint.') + log_string(f'not trying to load checkpoint.') else: try: @@ -454,13 +444,13 @@ else: nb_epochs_finished = checkpoint['nb_epochs_finished'] model.load_state_dict(checkpoint['model_state']) optimizer.load_state_dict(checkpoint['optimizer_state']) - log_string(f'Checkpoint loaded with {nb_epochs_finished} epochs finished.') + log_string(f'checkpoint loaded with {nb_epochs_finished} epochs finished.') except FileNotFoundError: - log_string('Starting from scratch.') + log_string('starting from scratch.') except: - log_string('Error when loading the checkpoint.') + log_string('error when loading the checkpoint.') exit(1) ###################################################################### @@ -471,9 +461,9 @@ token_count = 0 for input in task.batches(split = 'train'): token_count += F.one_hot(input, num_classes = task.vocabulary_size()).sum((0, 1)) token_probas = token_count / token_count.sum() -h = -torch.xlogy(token_probas, token_probas).sum() -train_set_perplexity = math.exp(h) -log_string(f'Train set perplexity {train_set_perplexity}') +entropy = -torch.xlogy(token_probas, token_probas).sum() +train_set_perplexity = math.exp(entropy) +#log_string(f'train set perplexity {train_set_perplexity}') for k in range(nb_epochs_finished, nb_epochs): @@ -508,7 +498,7 @@ for k in range(nb_epochs_finished, nb_epochs): train_perplexity = math.exp(min(100, acc_train_loss/nb_train_samples)) test_perplexity = math.exp(min(100, acc_test_loss/nb_test_samples)) - log_string(f'perplexity {k} train {train_perplexity} test {test_perplexity}') + log_string(f'perplexity {k} train_set {train_set_perplexity} train_prediction {train_perplexity} test_prediction {test_perplexity}') task.produce_results(k, model)