######################################################################
- errors[linear_layer].append((nb_hidden, test_error))
+ errors[linear_layer].append(
+ (nb_hidden, test_error * 100, acc_train_loss / train_input.size(0))
+ )
import matplotlib.pyplot as plt
-fig = plt.figure()
-fig.set_figheight(6)
-fig.set_figwidth(8)
-ax = fig.add_subplot(1, 1, 1)
+def save_fig(filename, ymax, ylabel, index):
+ fig = plt.figure()
+ fig.set_figheight(6)
+ fig.set_figwidth(8)
-ax.set_ylim(0, 1)
-ax.spines.right.set_visible(False)
-ax.spines.top.set_visible(False)
-ax.set_xscale("log")
-ax.set_xlabel("Nb hidden units")
-ax.set_ylabel("Test error (%)")
+ ax = fig.add_subplot(1, 1, 1)
-X = torch.tensor([x[0] for x in errors[nn.Linear]])
-Y = torch.tensor([x[1] for x in errors[nn.Linear]])
-ax.plot(X, Y, color="gray", label="nn.Linear")
+ ax.set_ylim(0, ymax)
+ ax.spines.right.set_visible(False)
+ ax.spines.top.set_visible(False)
+ ax.set_xscale("log")
+ ax.set_xlabel("Nb hidden units")
+ ax.set_ylabel(ylabel)
-X = torch.tensor([x[0] for x in errors[QLinear]])
-Y = torch.tensor([x[1] for x in errors[QLinear]])
-ax.plot(X, Y, color="red", label="QLinear")
+ X = torch.tensor([x[0] for x in errors[nn.Linear]])
+ Y = torch.tensor([x[index] for x in errors[nn.Linear]])
+ ax.plot(X, Y, color="gray", label="nn.Linear")
-ax.legend(frameon=False, loc=1)
+ X = torch.tensor([x[0] for x in errors[QLinear]])
+ Y = torch.tensor([x[index] for x in errors[QLinear]])
+ ax.plot(X, Y, color="red", label="QLinear")
-filename = f"bit_mlp.pdf"
-print(f"saving {filename}")
-fig.savefig(filename, bbox_inches="tight")
+ ax.legend(frameon=False, loc=1)
+
+ print(f"saving {filename}")
+ fig.savefig(filename, bbox_inches="tight")
+
+
+save_fig("bit_mlp_err.pdf", ymax=15, ylabel="Test error (%)", index=1)
+save_fig("bit_mlp_loss.pdf", ymax=1.25, ylabel="Train loss", index=2)
--- /dev/null
+#!/usr/bin/env python
+
+import math
+
+import torch, torchvision
+
+from torch import nn
+from torch.nn import functional as F
+
+torch.set_default_dtype(torch.float64)
+
+nb_hidden = 5
+hidden_dim = 100
+
+res = 256
+
+input = torch.cat(
+ [
+ torch.linspace(-1, 1, res)[None, :, None].expand(res, res, 1),
+ torch.linspace(-1, 1, res)[:, None, None].expand(res, res, 1),
+ ],
+ dim=-1,
+).reshape(-1, 2)
+
+
+class Angles(nn.Module):
+ def forward(self, x):
+ return x.clamp(min=-0.5, max=0.5)
+
+
+for activation in [nn.ReLU, nn.Tanh, nn.Softplus, Angles]:
+ for s in [1.0, 10.0]:
+ layers = [nn.Linear(2, hidden_dim), activation()]
+ for k in range(nb_hidden - 1):
+ layers += [nn.Linear(hidden_dim, hidden_dim), activation()]
+ layers += [nn.Linear(hidden_dim, 2)]
+ model = nn.Sequential(*layers)
+
+ with torch.no_grad():
+ for p in model.parameters():
+ p *= s
+
+ output = model(input)
+
+ img = (output[:, 1] - output[:, 0]).reshape(1, 1, res, res)
+
+ img = (img - img.mean()) / (1 * img.std())
+
+ img = img.clamp(min=-1, max=1)
+
+ img = torch.cat(
+ [
+ (1 + img).clamp(max=1),
+ (1 - img.abs()).clamp(min=0),
+ (1 - img).clamp(max=1),
+ ],
+ dim=1,
+ )
+
+ name_activation = {
+ nn.ReLU: "relu",
+ nn.Tanh: "tanh",
+ nn.Softplus: "softplus",
+ Angles: "angles",
+ }[activation]
+
+ torchvision.utils.save_image(img, f"result-{name_activation}-{s}.png")