import torch from torchvision import datasets import argparse import os import urllib ###################################################################### parser = argparse.ArgumentParser(description='DLC prologue file for practical sessions.') parser.add_argument('--full', action='store_true', default=False, help = 'Use the full set, can take ages (default False)') parser.add_argument('--tiny', action='store_true', default=False, help = 'Use a very small set for quick checks (default False)') parser.add_argument('--seed', type = int, default = 0, help = 'Random seed (default 0, < 0 is no seeding)') parser.add_argument('--cifar', action='store_true', default=False, help = 'Use the CIFAR data-set and not MNIST (default False)') parser.add_argument('--data_dir', type = str, default = None, help = 'Where are the PyTorch data located (default $PYTORCH_DATA_DIR or \'./data\')') # Timur's fix parser.add_argument('-f', '--file', help = 'quick hack for jupyter') args = parser.parse_args() if args.seed >= 0: torch.manual_seed(args.seed) ###################################################################### # The data def convert_to_one_hot_labels(input, target): tmp = input.new_zeros(target.size(0), target.max() + 1) tmp.scatter_(1, target.view(-1, 1), 1.0) return tmp def load_data(cifar = None, one_hot_labels = False, normalize = False, flatten = True): if args.data_dir is not None: data_dir = args.data_dir else: data_dir = os.environ.get('PYTORCH_DATA_DIR') if data_dir is None: data_dir = './data' if args.cifar or (cifar is not None and cifar): print('* Using CIFAR') cifar_train_set = datasets.CIFAR10(data_dir + '/cifar10/', train = True, download = True) cifar_test_set = datasets.CIFAR10(data_dir + '/cifar10/', train = False, download = True) train_input = torch.from_numpy(cifar_train_set.data) train_input = train_input.transpose(3, 1).transpose(2, 3).float() train_target = torch.tensor(cifar_train_set.targets, dtype = torch.int64) test_input = torch.from_numpy(cifar_test_set.data).float() test_input = test_input.transpose(3, 1).transpose(2, 3).float() test_target = torch.tensor(cifar_test_set.targets, dtype = torch.int64) else: print('* Using MNIST') ###################################################################### # import torchvision # raw_folder = data_dir + '/mnist/raw/' # resources = [ # ("https://fleuret.org/dlc/data/train-images-idx3-ubyte.gz", "f68b3c2dcbeaaa9fbdd348bbdeb94873"), # ("https://fleuret.org/dlc/data/train-labels-idx1-ubyte.gz", "d53e105ee54ea40749a09fcbcd1e9432"), # ("https://fleuret.org/dlc/data/t10k-images-idx3-ubyte.gz", "9fb629c4189551a2d022fa330f9573f3"), # ("https://fleuret.org/dlc/data/t10k-labels-idx1-ubyte.gz", "ec29112dd5afa0611ce80d1b7f02629c") # ] # os.makedirs(raw_folder, exist_ok=True) # # download files # for url, md5 in resources: # filename = url.rpartition('/')[2] # torchvision.datasets.utils.download_and_extract_archive(url, download_root=raw_folder, filename=filename, md5=md5) ###################################################################### mnist_train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True) mnist_test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True) train_input = mnist_train_set.data.view(-1, 1, 28, 28).float() train_target = mnist_train_set.targets test_input = mnist_test_set.data.view(-1, 1, 28, 28).float() test_target = mnist_test_set.targets if flatten: train_input = train_input.clone().reshape(train_input.size(0), -1) test_input = test_input.clone().reshape(test_input.size(0), -1) if args.full: if args.tiny: raise ValueError('Cannot have both --full and --tiny') else: if args.tiny: print('** Reduce the data-set to the tiny setup') train_input = train_input.narrow(0, 0, 500) train_target = train_target.narrow(0, 0, 500) test_input = test_input.narrow(0, 0, 100) test_target = test_target.narrow(0, 0, 100) else: print('** Reduce the data-set (use --full for the full thing)') train_input = train_input.narrow(0, 0, 1000) train_target = train_target.narrow(0, 0, 1000) test_input = test_input.narrow(0, 0, 1000) test_target = test_target.narrow(0, 0, 1000) print('** Use {:d} train and {:d} test samples'.format(train_input.size(0), test_input.size(0))) if one_hot_labels: train_target = convert_to_one_hot_labels(train_input, train_target) test_target = convert_to_one_hot_labels(test_input, test_target) if normalize: mu, std = train_input.mean(), train_input.std() train_input.sub_(mu).div_(std) test_input.sub_(mu).div_(std) return train_input, train_target, test_input, test_target ###################################################################### def mnist_to_pairs(nb, input, target): input = torch.functional.F.avg_pool2d(input, kernel_size = 2) a = torch.randperm(input.size(0)) a = a[:2 * nb].view(nb, 2) input = torch.cat((input[a[:, 0]], input[a[:, 1]]), 1) classes = target[a] target = (classes[:, 0] <= classes[:, 1]).long() return input, target, classes ###################################################################### def generate_pair_sets(nb): if args.data_dir is not None: data_dir = args.data_dir else: data_dir = os.environ.get('PYTORCH_DATA_DIR') if data_dir is None: data_dir = './data' train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True) train_input = train_set.data.view(-1, 1, 28, 28).float() train_target = train_set.targets test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True) test_input = test_set.data.view(-1, 1, 28, 28).float() test_target = test_set.targets return mnist_to_pairs(nb, train_input, train_target) + \ mnist_to_pairs(nb, test_input, test_target) ######################################################################