From: François Fleuret <francois@fleuret.org>
Date: Fri, 13 Oct 2023 11:51:34 +0000 (+0200)
Subject: Update.
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=f44ab6863f93ae348e66ffbf52251d96d3b5453c;p=culture.git

Update.
---

diff --git a/qmlp.y b/qmlp.py
similarity index 54%
rename from qmlp.y
rename to qmlp.py
index 7b97edb..e12f0e1 100755
--- a/qmlp.y
+++ b/qmlp.py
@@ -39,40 +39,24 @@ def dequantize(q, xmin, xmax):
 ######################################################################
 
 
-def create_model():
-    hidden_dim = 32
-
-    model = nn.Sequential(
-        nn.Linear(2, hidden_dim),
-        nn.ReLU(),
-        nn.Linear(hidden_dim, hidden_dim),
-        nn.ReLU(),
-        nn.Linear(hidden_dim, 2),
-    )
-
-    return model
-
-
-######################################################################
 
 
 def generate_sets_and_params(
-    nb_mlps,
+    batch_nb_mlps,
     nb_samples,
     batch_size,
     nb_epochs,
     device=torch.device("cpu"),
     print_log=False,
 ):
-    data_input = torch.zeros(nb_mlps, 2 * nb_samples, 2, device=device)
+    data_input = torch.zeros(batch_nb_mlps, 2 * nb_samples, 2, device=device)
     data_targets = torch.zeros(
-        nb_mlps, 2 * nb_samples, dtype=torch.int64, device=device
+        batch_nb_mlps, 2 * nb_samples, dtype=torch.int64, device=device
     )
 
     while (data_targets.float().mean(-1) - 0.5).abs().max() > 0.1:
         i = (data_targets.float().mean(-1) - 0.5).abs() > 0.1
         nb = i.sum()
-        print(f"{nb=}")
 
         nb_rec = 2
         support = torch.rand(nb, nb_rec, 2, 3, device=device) * 2 - 1
@@ -108,10 +92,10 @@ def generate_sets_and_params(
     test_targets = test_targets
 
     hidden_dim = 32
-    w1 = torch.randn(nb_mlps, hidden_dim, 2, device=device) / math.sqrt(2)
-    b1 = torch.zeros(nb_mlps, hidden_dim, device=device)
-    w2 = torch.randn(nb_mlps, 2, hidden_dim, device=device) / math.sqrt(hidden_dim)
-    b2 = torch.zeros(nb_mlps, 2, device=device)
+    w1 = torch.randn(batch_nb_mlps, hidden_dim, 2, device=device) / math.sqrt(2)
+    b1 = torch.zeros(batch_nb_mlps, hidden_dim, device=device)
+    w2 = torch.randn(batch_nb_mlps, 2, hidden_dim, device=device) / math.sqrt(hidden_dim)
+    b2 = torch.zeros(batch_nb_mlps, 2, device=device)
 
     w1.requires_grad_()
     b1.requires_grad_()
@@ -158,13 +142,13 @@ def generate_sets_and_params(
         # print(f"{k=} {acc_train_loss=} {train_error=}")
 
     q_params = torch.cat(
-        [quantize(p.view(nb_mlps, -1), -2, 2) for p in [w1, b1, w2, b2]], dim=1
+        [quantize(p.view(batch_nb_mlps, -1), -2, 2) for p in [w1, b1, w2, b2]], dim=1
     )
     q_train_set = torch.cat([q_train_input, train_targets[:, :, None]], -1).reshape(
-        nb_mlps, -1
+        batch_nb_mlps, -1
     )
     q_test_set = torch.cat([q_test_input, test_targets[:, :, None]], -1).reshape(
-        nb_mlps, -1
+        batch_nb_mlps, -1
     )
 
     return q_train_set, q_test_set, q_params
@@ -173,51 +157,59 @@ def generate_sets_and_params(
 ######################################################################
 
 
-def evaluate_q_params(q_params, q_set, batch_size=25, device=torch.device("cpu")):
+def evaluate_q_params(q_params, q_set, batch_size=25, device=torch.device("cpu"), nb_mlps_per_batch=1024):
+
+    errors = []
     nb_mlps = q_params.size(0)
-    hidden_dim = 32
-    w1 = torch.empty(nb_mlps, hidden_dim, 2, device=device)
-    b1 = torch.empty(nb_mlps, hidden_dim, device=device)
-    w2 = torch.empty(nb_mlps, 2, hidden_dim, device=device)
-    b2 = torch.empty(nb_mlps, 2, device=device)
-
-    with torch.no_grad():
-        k = 0
-        for p in [w1, b1, w2, b2]:
-            print(f"{p.size()=}")
-            x = dequantize(q_params[:, k : k + p.numel() // nb_mlps], -2, 2).view(
-                p.size()
-            )
-            p.copy_(x)
-            k += p.numel() // nb_mlps
 
-    q_set = q_set.view(nb_mlps, -1, 3)
-    data_input = dequantize(q_set[:, :, :2], -1, 1).to(device)
-    data_targets = q_set[:, :, 2].to(device)
+    for n in range(0,nb_mlps,nb_mlps_per_batch):
+        batch_nb_mlps = min(nb_mlps_per_batch,nb_mlps-n)
+        batch_q_params = q_params[n:n+batch_nb_mlps]
+        batch_q_set = q_set[n:n+batch_nb_mlps]
+        hidden_dim = 32
+        w1 = torch.empty(batch_nb_mlps, hidden_dim, 2, device=device)
+        b1 = torch.empty(batch_nb_mlps, hidden_dim, device=device)
+        w2 = torch.empty(batch_nb_mlps, 2, hidden_dim, device=device)
+        b2 = torch.empty(batch_nb_mlps, 2, device=device)
 
-    print(f"{data_input.size()=} {data_targets.size()=}")
+        with torch.no_grad():
+            k = 0
+            for p in [w1, b1, w2, b2]:
+                print(f"{p.size()=}")
+                x = dequantize(batch_q_params[:, k : k + p.numel() // batch_nb_mlps], -2, 2).view(
+                    p.size()
+                )
+                p.copy_(x)
+                k += p.numel() // batch_nb_mlps
 
-    criterion = nn.CrossEntropyLoss()
-    criterion.to(device)
+        batch_q_set = batch_q_set.view(batch_nb_mlps, -1, 3)
+        data_input = dequantize(batch_q_set[:, :, :2], -1, 1).to(device)
+        data_targets = batch_q_set[:, :, 2].to(device)
+
+        print(f"{data_input.size()=} {data_targets.size()=}")
+
+        criterion = nn.CrossEntropyLoss()
+        criterion.to(device)
+
+        acc_loss = 0.0
+        nb_errors = 0
 
-    acc_loss = 0.0
-    nb_errors = 0
+        for input, targets in zip(
+            data_input.split(batch_size, dim=1), data_targets.split(batch_size, dim=1)
+        ):
+            h = torch.einsum("mij,mnj->mni", w1, input) + b1[:, None, :]
+            h = F.relu(h)
+            output = torch.einsum("mij,mnj->mni", w2, h) + b2[:, None, :]
+            loss = F.cross_entropy(output.reshape(-1, output.size(-1)), targets.reshape(-1))
+            acc_loss += loss.item() * input.size(0)
+            wta = output.argmax(-1)
+            nb_errors += (wta != targets).long().sum(-1)
 
-    for input, targets in zip(
-        data_input.split(batch_size, dim=1), data_targets.split(batch_size, dim=1)
-    ):
-        h = torch.einsum("mij,mnj->mni", w1, input) + b1[:, None, :]
-        h = F.relu(h)
-        output = torch.einsum("mij,mnj->mni", w2, h) + b2[:, None, :]
-        loss = F.cross_entropy(output.reshape(-1, output.size(-1)), targets.reshape(-1))
-        acc_loss += loss.item() * input.size(0)
-        wta = output.argmax(-1)
-        nb_errors += (wta != targets).long().sum(-1)
+        errors.append(nb_errors / data_input.size(1))
+        acc_loss = acc_loss / data_input.size(1)
 
-    error = nb_errors / data_input.size(1)
-    acc_loss = acc_loss / data_input.size(1)
 
-    return error
+    return torch.cat(errors)
 
 
 ######################################################################
@@ -229,40 +221,41 @@ def generate_sequence_and_test_set(
     batch_size,
     nb_epochs,
     device,
+    nb_mlps_per_batch=1024,
 ):
-    q_train_set, q_test_set, q_params = generate_sets_and_params(
-        nb_mlps,
-        nb_samples,
-        batch_size,
-        nb_epochs,
-        device=device,
-    )
 
-    input = torch.cat(
-        [
-            q_train_set,
-            q_train_set.new_full(
-                (
-                    q_train_set.size(0),
-                    1,
+    inputs, q_test_sets = [],[]
+
+    for n in range(0,nb_mlps,nb_mlps_per_batch):
+        q_train_set, q_test_set, q_params = generate_sets_and_params(
+            batch_nb_mlps = min(nb_mlps_per_batch, nb_mlps - n),
+            nb_samples=nb_samples,
+            batch_size=batch_size,
+            nb_epochs=nb_epochs,
+            device=device,
+        )
+
+        inputs.append(torch.cat(
+            [
+                q_train_set,
+                q_train_set.new_full(
+                    (
+                        q_train_set.size(0),
+                        1,
+                    ),
+                    nb_quantization_levels,
                 ),
-                nb_quantization_levels,
-            ),
-            q_params,
-        ],
-        dim=-1,
-    )
+                q_params,
+            ],
+            dim=-1,
+        ))
 
-    print(f"SANITY #1 {q_train_set.size()=} {q_params.size()=} {input.size()=}")
+        q_test_sets.append(q_test_set)
 
-    ar_mask = (
-        (torch.arange(input.size(0), device=input.device) > q_train_set.size(0) + 1)
-        .long()
-        .view(1, -1)
-        .reshape(nb_mlps, -1)
-    )
+    input = torch.cat(inputs)
+    q_test_set = torch.cat(q_test_sets)
 
-    return input, ar_mask, q_test_set
+    return input, q_test_set
 
 
 ######################################################################
@@ -270,7 +263,7 @@ def generate_sequence_and_test_set(
 if __name__ == "__main__":
     import time
 
-    nb_mlps, nb_samples = 128, 200
+    batch_nb_mlps, nb_samples = 128, 500
 
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
@@ -278,26 +271,22 @@ if __name__ == "__main__":
 
     data = []
 
-    for n in range(2):
-        data.append(
-            generate_sequence_and_test_set(
-                nb_mlps=nb_mlps,
-                nb_samples=nb_samples,
-                device=device,
-                batch_size=25,
-                nb_epochs=250,
-            )
-        )
+    input, q_test_set = generate_sequence_and_test_set(
+        nb_mlps=batch_nb_mlps,
+        nb_samples=nb_samples,
+        device=device,
+        batch_size=25,
+        nb_epochs=250,
+        nb_mlps_per_batch=17
+    )
 
     end_time = time.perf_counter()
-    nb = sum([i.size(0) for i, _, _ in data])
-    print(f"{nb / (end_time - start_time):.02f} samples per second")
-
-    for input, ar_mask, q_test_set in data:
-        q_train_set = input[:, : nb_samples * 3]
-        q_params = input[:, nb_samples * 3 + 1 :]
-        print(f"SANITY #2 {q_train_set.size()=} {q_params.size()=} {input.size()=}")
-        error_train = evaluate_q_params(q_params, q_train_set)
-        print(f"train {error_train*100}%")
-        error_test = evaluate_q_params(q_params, q_test_set)
-        print(f"test {error_test*100}%")
+    print(f"{input.size(0) / (end_time - start_time):.02f} samples per second")
+
+    q_train_set = input[:, : nb_samples * 3]
+    q_params = input[:, nb_samples * 3 + 1 :]
+    print(f"SANITY #2 {q_train_set.size()=} {q_params.size()=} {input.size()=}")
+    error_train = evaluate_q_params(q_params, q_train_set, nb_mlps_per_batch=17)
+    print(f"train {error_train*100}%")
+    error_test = evaluate_q_params(q_params, q_test_set, nb_mlps_per_batch=17)
+    print(f"test {error_test*100}%")
diff --git a/tasks.py b/tasks.py
index 183c3cf..ea10d7c 100755
--- a/tasks.py
+++ b/tasks.py
@@ -1550,3 +1550,105 @@ class Grid(Task):
 
 
 ######################################################################
+
+import qmlp
+
+
+class QMLP(Task):
+
+    ######################
+
+    def __init__(
+        self,
+        nb_train_samples,
+        nb_test_samples,
+        batch_size,
+        logger=None,
+        device=torch.device("cpu"),
+    ):
+        super().__init__()
+
+        self.device = device
+        self.batch_size = batch_size
+
+        if logger is not None:
+            logger(
+                f"generating {nb_train_samples+nb_test_samples} samples (can take some time)"
+            )
+
+        self.train_descr = self.grid_factory.generate_samples(
+            nb_train_samples, lambda r: tqdm.tqdm(r)
+        )
+        self.test_descr = self.grid_factory.generate_samples(
+            nb_test_samples, lambda r: tqdm.tqdm(r)
+        )
+
+        # Build the tokenizer
+        tokens = set()
+        for d in [self.train_descr, self.test_descr]:
+            for s in d:
+                for t in s.strip().split(" "):
+                    tokens.add(t)
+        # make this set a sorted list to get the same tensors given
+        # the same descr
+        tokens = list(tokens)
+        tokens.sort()
+        tokens = ["#"] + tokens
+        self.token2id = dict([(t, n) for n, t in enumerate(tokens)])
+        self.id2token = dict([(n, t) for n, t in enumerate(tokens)])
+        self.t_nul = self.token2id["#"]
+        self.t_true = self.token2id["true"]
+        self.t_false = self.token2id["false"]
+
+        # Tokenize the train and test sets
+        self.train_input = self.str2tensor(self.train_descr)
+        self.test_input = self.str2tensor(self.test_descr)
+
+    def batches(self, split="train"):
+        assert split in {"train", "test"}
+        input = self.train_input if split == "train" else self.test_input
+        for batch in tqdm.tqdm(
+            input.split(self.batch_size), dynamic_ncols=True, desc=f"epoch-{split}"
+        ):
+            yield self.trim(batch)
+
+    def vocabulary_size(self):
+        return len(self.token2id)
+
+    def produce_results(
+        self, n_epoch, model, result_dir, logger, deterministic_synthesis
+    ):
+        correct = self.test_input[:1000]
+        result = correct.clone()
+        ar_mask = torch.logical_or(result == self.t_true, result == self.t_false).long()
+        result *= 1 - ar_mask  # paraaaaanoiaaaaaaa
+
+        logger(f"----------------------------------------------------------")
+
+        for e in self.tensor2str(result[:10]):
+            logger(f"test_before {e}")
+
+        masked_inplace_autoregression(
+            model,
+            self.batch_size,
+            result,
+            ar_mask,
+            deterministic_synthesis,
+            device=self.device,
+        )
+
+        logger(f"----------------------------------------------------------")
+
+        for e in self.tensor2str(result[:10]):
+            logger(f"test_after  {e}")
+
+        logger(f"----------------------------------------------------------")
+
+        nb_total = ar_mask.sum().item()
+        nb_correct = ((correct == result).long() * ar_mask).sum().item()
+
+        logger(f"test_performance {n_epoch} {nb_total=} {nb_correct=}")
+        logger(f"main_test_accuracy {n_epoch} {nb_correct / nb_total}")
+
+
+######################################################################