From: François Fleuret Date: Mon, 20 Mar 2023 07:30:02 +0000 (+0100) Subject: Update X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=280f38363e34e202d38e6f7c00288329ab067a81;p=beaver.git Update --- diff --git a/beaver.py b/beaver.py index de38ff4..e7decd1 100755 --- a/beaver.py +++ b/beaver.py @@ -81,6 +81,8 @@ parser.add_argument("--maze_width", type=int, default=21) parser.add_argument("--maze_nb_walls", type=int, default=15) +parser.add_argument("--oneshot_mode", type=str, default="head") + ###################################################################### args = parser.parse_args() @@ -172,8 +174,7 @@ def compute_perplexity(model, split="train"): def one_shot(gpt, task): t = gpt.training gpt.eval() - mode='head' - dim_in=args.dim_model * (args.nb_blocks * 2 if mode=='deep' else 1) + dim_in = args.dim_model * (args.nb_blocks * 2 if args.oneshot_mode == "deep" else 1) model = nn.Sequential( nn.Linear(dim_in, args.dim_model), nn.ReLU(), @@ -188,7 +189,7 @@ def one_shot(gpt, task): acc_train_loss, nb_train_samples = 0, 0 for input, targets in task.policy_batches(split="train"): - output_gpt = gpt(mygpt.BracketedSequence(input), mode=mode).x + output_gpt = gpt(mygpt.BracketedSequence(input), mode=args.oneshot_mode).x output = model(output_gpt) targets = targets * (input.unsqueeze(-1) == maze.v_empty) output = output * (input.unsqueeze(-1) == maze.v_empty) @@ -206,7 +207,7 @@ def one_shot(gpt, task): acc_test_loss, nb_test_samples = 0, 0 for input, targets in task.policy_batches(split="test"): - output_gpt = gpt(mygpt.BracketedSequence(input), mode=mode).x + output_gpt = gpt(mygpt.BracketedSequence(input), mode=args.oneshot_mode).x output = model(output_gpt) targets = targets * (input.unsqueeze(-1) == maze.v_empty) output = output * (input.unsqueeze(-1) == maze.v_empty) @@ -225,7 +226,7 @@ def one_shot(gpt, task): # ------------------- input = task.test_input[:32, : task.height * task.width] targets = task.test_policies[:32] - output_gpt = gpt(mygpt.BracketedSequence(input), mode=mode).x + output_gpt = gpt(mygpt.BracketedSequence(input), mode=args.oneshot_mode).x output = model(output_gpt) # losses = (-output.log_softmax(-1) * targets + targets.xlogy(targets)).sum(-1) # losses = losses * (input == maze.v_empty) @@ -238,7 +239,9 @@ def one_shot(gpt, task): losses = losses.reshape(-1, args.maze_height, args.maze_width) input = input.reshape(-1, args.maze_height, args.maze_width) maze.save_image( - os.path.join(args.result_dir, f"oneshot_{n_epoch:04d}.png"), + os.path.join( + args.result_dir, f"oneshot_{args.oneshot_mode}_{n_epoch:04d}.png" + ), mazes=input, score_paths=losses, ) diff --git a/mygpt.py b/mygpt.py index a1db2e3..0b63ac8 100755 --- a/mygpt.py +++ b/mygpt.py @@ -246,19 +246,19 @@ class MyGPT(nn.Module): m.bias.zero_() m.weight.fill_(1.0) - def forward(self, bs, mode='standard'): + def forward(self, bs, mode="standard"): bs.x = F.pad(bs.x, (1, -1)) bs = self.embedding(bs) - if mode=='standard': + if mode == "standard": bs = self.trunk(bs) bs = self.readout(bs) - elif mode=='head': + elif mode == "head": bs = self.trunk(bs) - elif mode=='deep': + elif mode == "deep": r = [] for l in self.trunk: bs = l(bs) - r += [ bs.slice() ] + r += [bs.slice()] bs = BracketedSequence(torch.cat(r, -1)) else: raise ValueError