X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;ds=inline;f=tasks.py;h=1a6c41572ee8c5e2f5e8ce65b9d8cbfea3834f8a;hb=9e3211bab93700003ff835e346ef413044147b73;hp=29f1e5a783024a3948867d2ffc0dae8c71b09954;hpb=2be22c9825d8aebe8d184e9501355a31318abf2b;p=culture.git

diff --git a/tasks.py b/tasks.py
index 29f1e5a..1a6c415 100755
--- a/tasks.py
+++ b/tasks.py
@@ -5,7 +5,7 @@
 
 # Written by Francois Fleuret <francois@fleuret.org>
 
-import math, os, tqdm
+import math, os, tqdm, warnings
 
 import torch, torchvision
 
@@ -63,7 +63,7 @@ def masked_inplace_autoregression(
 
 
 class Task:
-    def batches(self, split="train"):
+    def batches(self, split="train", nb_to_use=-1, desc=None):
         pass
 
     def vocabulary_size(self):
@@ -489,7 +489,7 @@ class PicoCLVR(Task):
         self.train_input = self.tensorize(self.train_descr)
         self.test_input = self.tensorize(self.test_descr)
 
-    def batches(self, split="train"):
+    def batches(self, split="train", nb_to_use=-1, desc=None):
         assert split in {"train", "test"}
         input = self.train_input if split == "train" else self.test_input
         for batch in tqdm.tqdm(
@@ -754,15 +754,17 @@ class Maze(Task):
     def compute_error(
         self, model, split="train", nb_to_use=-1, deterministic_synthesis=False
     ):
+        model_device = next(model.parameters()).device
         nb_total, nb_correct = 0, 0
         count = torch.zeros(
             self.width * self.height,
             self.width * self.height,
-            device=self.device,
+            device=model_device,
             dtype=torch.int64,
         )
 
         for input in self.batches(split, nb_to_use):
+            input = input.to(model_device)
             result = input.clone()
             ar_mask = result.new_zeros(result.size())
             ar_mask[:, self.height * self.width :] = 1
@@ -836,7 +838,7 @@ class Maze(Task):
                         eol = " " if j < count.size(1) - 1 else "\n"
                         f.write(f"{count[i,j]}{eol}")
 
-        input = self.test_input[:48]
+        input = self.test_input[:48].to(next(model.parameters()).device)
         result = input.clone()
         ar_mask = result.new_zeros(result.size())
         ar_mask[:, self.height * self.width :] = 1
@@ -1098,6 +1100,34 @@ class Stack(Task):
             device=self.device,
         )
 
+        #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        for label, input in [
+            ("train", self.train_input[:32]),
+            ("test", self.test_input[:32]),
+        ]:
+            output = model(BracketedSequence(input)).x
+            output = output.log_softmax(dim=-1)
+            filename = os.path.join(
+                result_dir, f"stack_with_crossentropy_{n_epoch:04d}_{label}.txt"
+            )
+            with open(filename, "w") as f:
+                for n in range(input.size(0)):
+                    s = stack.seq_to_str(
+                        input[n], nb_stacks=self.nb_stacks, nb_digits=self.nb_digits
+                    )
+                    for t, k, w in zip(range(input[n].size(0)), input[n], s.split(" ")):
+                        u = (
+                            " " * (10 - len(w))
+                            + w
+                            + " "
+                            + str(output[n][t][k].exp().item())
+                            + "\n"
+                        )
+                        f.write(u)
+                    f.write("\n")
+            logger(f"wrote {filename}")
+        #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
         for n in range(result.size(0)):
             logger(
                 f"test_after  {stack.seq_to_str(result[n],nb_stacks=self.nb_stacks,nb_digits=self.nb_digits)}"
@@ -1685,7 +1715,7 @@ class Grid(Task):
         self.t_nul = self.token2id["#"]
         self.t_true = self.token2id["true"]
         self.t_false = self.token2id["false"]
-        self.t_pipe = self.token2id["|"]
+        # self.t_pipe = self.token2id["|"]
 
         # Tokenize the train and test sets
         self.train_input = self.str2tensor(self.train_descr)
@@ -1694,7 +1724,7 @@ class Grid(Task):
             None if len(self.play_descr) == 0 else self.str2tensor(self.play_descr)
         )
 
-    def batches(self, split="train"):
+    def batches(self, split="train", nb_to_use=-1, desc=None):
         assert split in {"train", "test"}
         input = self.train_input if split == "train" else self.test_input
         for batch in tqdm.tqdm(
@@ -1823,7 +1853,7 @@ class QMLP(Task):
 
         self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1
 
-    def batches(self, split="train"):
+    def batches(self, split="train", nb_to_use=-1, desc=None):
         assert split in {"train", "test"}
         input = self.train_input if split == "train" else self.test_input
         for batch in tqdm.tqdm(
@@ -1867,10 +1897,10 @@ class QMLP(Task):
 
 ######################################################################
 
-import escape
+import greed
 
 
-class Escape(Task):
+class Greed(Task):
     def __init__(
         self,
         nb_train_samples,
@@ -1880,6 +1910,7 @@ class Escape(Task):
         width,
         T,
         nb_walls,
+        nb_coins,
         logger=None,
         device=torch.device("cpu"),
     ):
@@ -1887,18 +1918,27 @@ class Escape(Task):
 
         self.batch_size = batch_size
         self.device = device
-        self.height = height
-        self.width = width
 
-        states, actions, rewards = escape.generate_episodes(
-            nb_train_samples + nb_test_samples, height, width, T, nb_walls
+        self.world = greed.GreedWorld(height, width, T, nb_walls, nb_coins)
+
+        states, actions, rewards = self.world.generate_episodes(
+            nb_train_samples + nb_test_samples
         )
-        seq = escape.episodes2seq(states, actions, rewards, lookahead_delta=T)
-        # seq = seq[:, seq.size(1) // 3 : 2 * seq.size(1) // 3]
+        seq = self.world.episodes2seq(states, actions, rewards)
         self.train_input = seq[:nb_train_samples].to(self.device)
         self.test_input = seq[nb_train_samples:].to(self.device)
 
-        self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1
+    def wipe_lookahead_rewards(self, batch):
+        t = torch.arange(batch.size(1), device=batch.device)[None, :]
+        u = torch.randint(batch.size(1), (batch.size(0), 1), device=batch.device)
+        lr_mask = (t <= u).long() * (
+            t % self.world.it_len == self.world.index_lookahead_reward
+        ).long()
+
+        return (
+            lr_mask * self.world.lookahead_reward2code(greed.REWARD_UNKNOWN)
+            + (1 - lr_mask) * batch
+        )
 
     def batches(self, split="train", nb_to_use=-1, desc=None):
         assert split in {"train", "test"}
@@ -1910,22 +1950,15 @@ class Escape(Task):
         for batch in tqdm.tqdm(
             input.split(self.batch_size), dynamic_ncols=True, desc=desc
         ):
-            yield batch
+            yield self.wipe_lookahead_rewards(batch)
 
     def vocabulary_size(self):
-        return self.nb_codes
+        return self.world.nb_codes
 
     def thinking_autoregression(
         self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000
     ):
-        result = self.test_input[:250].clone()
-        t = torch.arange(result.size(1), device=result.device)[None, :]
-
-        state_len = self.height * self.width
-        index_action = state_len
-        index_reward = state_len + 1
-        index_lookahead_reward = state_len + 2
-        it_len = state_len + 3  # state / action / reward / lookahead_reward
+        snapshots = []
 
         def ar(result, ar_mask, logit_biases=None):
             ar_mask = ar_mask.expand_as(result)
@@ -1940,42 +1973,68 @@ class Escape(Task):
                 device=self.device,
                 progress_bar_desc=None,
             )
+            warnings.warn("keeping thinking snapshots", RuntimeWarning)
+            snapshots.append(result[:100].detach().clone())
 
         # Generate iteration after iteration
 
-        optimistic_bias = result.new_zeros(self.nb_codes, device=result.device)
-        optimistic_bias[escape.lookahead_reward2code(-1)] = -math.log(1e1)
-        optimistic_bias[escape.lookahead_reward2code(1)] = math.log(1e1)
+        result = self.test_input[:250].clone()
+        # Erase all the content but that of the first iteration
+        result[:, self.world.it_len :] = -1
+        # Set the lookahead_reward of the firs to UNKNOWN
+        result[:, self.world.index_lookahead_reward] = self.world.lookahead_reward2code(
+            greed.REWARD_UNKNOWN
+        )
+
+        t = torch.arange(result.size(1), device=result.device)[None, :]
 
         for u in tqdm.tqdm(
-            range(it_len, result.size(1) - it_len + 1, it_len), desc="thinking"
+            range(0, result.size(1), self.world.it_len),
+            desc="thinking",
         ):
-            # Re-generate the lookahead_reward pessimistically in the
-            # previous iterations
-            ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long()
-            ar(result, ar_mask, logit_biases=-optimistic_bias)
-
-            # Generate the state
-            ar_mask = (t >= u).long() * (t < u + state_len).long()
+            # Generate the next state but keep the initial one, the
+            # lookahead_reward of previous iterations are set to
+            # UNKNOWN
+            if u > 0:
+                result[
+                    :, u + self.world.index_lookahead_reward
+                ] = self.world.lookahead_reward2code(greed.REWARD_UNKNOWN)
+                ar_mask = (t >= u + self.world.index_states).long() * (
+                    t < u + self.world.index_states + self.world.state_len
+                ).long()
+                ar(result, ar_mask)
+
+            # Generate the action and reward with lookahead_reward to +1
+            result[
+                :, u + self.world.index_lookahead_reward
+            ] = self.world.lookahead_reward2code(greed.REWARD_PLUS)
+            ar_mask = (t >= u + self.world.index_reward).long() * (
+                t <= u + self.world.index_action
+            ).long()
             ar(result, ar_mask)
 
-            # Re-generate the lookahead_reward optimistically in the
-            # previous iterations
-            ar_mask = (t < u).long() * (t % it_len == index_lookahead_reward).long()
-            ar(result, ar_mask, logit_biases=optimistic_bias)
+            # Set the lookahead_reward to UNKNOWN for the next iterations
+            result[
+                :, u + self.world.index_lookahead_reward
+            ] = self.world.lookahead_reward2code(greed.REWARD_UNKNOWN)
 
-            # Generate the action and reward
-            ar_mask = (t >= u + index_action).long() * (t <= u + index_reward).long()
-            ar(result, ar_mask)
+        filename = os.path.join(result_dir, f"test_thinking_compute_{n_epoch:04d}.txt")
+        with open(filename, "w") as f:
+            for n in range(snapshots[0].size(0)):
+                for s in snapshots:
+                    lr, s, a, r = self.world.seq2episodes(
+                        s[n : n + 1],
+                    )
+                    str = self.world.episodes2str(
+                        lr, s, a, r, unicode=True, ansi_colors=True
+                    )
+                    f.write(str)
+                f.write("\n\n")
 
         # Saving the generated sequences
 
-        s, a, r, lr = escape.seq2episodes(
-            result, self.height, self.width, lookahead=True
-        )
-        str = escape.episodes2str(
-            s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True
-        )
+        lr, s, a, r = self.world.seq2episodes(result)
+        str = self.world.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True)
 
         filename = os.path.join(result_dir, f"test_thinking_seq_{n_epoch:04d}.txt")
         with open(filename, "w") as f:
@@ -1985,16 +2044,14 @@ class Escape(Task):
     def produce_results(
         self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000
     ):
-        result = self.test_input[:250].clone()
+        result = self.wipe_lookahead_rewards(self.test_input[:250].clone())
 
         # Saving the ground truth
 
-        s, a, r, lr = escape.seq2episodes(
-            result, self.height, self.width, lookahead=True
-        )
-        str = escape.episodes2str(
-            s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True
+        lr, s, a, r = self.world.seq2episodes(
+            result,
         )
+        str = self.world.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True)
 
         filename = os.path.join(result_dir, f"test_true_seq_{n_epoch:04d}.txt")
         with open(filename, "w") as f:
@@ -2004,8 +2061,7 @@ class Escape(Task):
         # Re-generating from the first frame
 
         ar_mask = (
-            torch.arange(result.size(1), device=result.device)
-            >= self.height * self.width + 3
+            torch.arange(result.size(1), device=result.device) >= self.world.it_len
         ).long()[None, :]
         ar_mask = ar_mask.expand_as(result)
         result *= 1 - ar_mask  # paraaaaanoiaaaaaaa
@@ -2021,12 +2077,10 @@ class Escape(Task):
 
         # Saving the generated sequences
 
-        s, a, r, lr = escape.seq2episodes(
-            result, self.height, self.width, lookahead=True
-        )
-        str = escape.episodes2str(
-            s, a, r, lookahead_rewards=lr, unicode=True, ansi_colors=True
+        lr, s, a, r = self.world.seq2episodes(
+            result,
         )
+        str = self.world.episodes2str(lr, s, a, r, unicode=True, ansi_colors=True)
 
         filename = os.path.join(result_dir, f"test_seq_{n_epoch:04d}.txt")
         with open(filename, "w") as f:
@@ -2039,3 +2093,190 @@ class Escape(Task):
 
 
 ######################################################################
+######################################################################
+
+import world
+
+
+class World(Task):
+    def save_image(self, input, result_dir, filename, logger):
+        img = world.sample2img(self.train_input.to("cpu"), self.height, self.width)
+        image_name = os.path.join(result_dir, filename)
+        torchvision.utils.save_image(img.float() / 255.0, image_name, nrow=8, padding=2)
+        logger(f"wrote {image_name}")
+
+    def __init__(
+        self,
+        nb_train_samples,
+        nb_test_samples,
+        batch_size,
+        result_dir=None,
+        logger=None,
+        device=torch.device("cpu"),
+    ):
+        super().__init__()
+
+        self.batch_size = batch_size
+        self.device = device
+        self.height = 6
+        self.width = 8
+
+        self.train_input = world.generate(
+            nb_train_samples, height=self.height, width=self.width
+        )
+        self.train_ar_mask = (
+            (torch.arange(self.train_input.size(1)) > self.train_input.size(1) // 2)
+            .long()[None, :]
+            .expand_as(self.train_input)
+        )
+
+        self.test_input = world.generate(
+            nb_test_samples, height=self.height, width=self.width
+        )
+        self.test_ar_mask = (
+            (torch.arange(self.test_input.size(1)) > self.test_input.size(1) // 2)
+            .long()[None, :]
+            .expand_as(self.test_input)
+        )
+
+        self.train_input, self.train_ar_mask = self.train_input.to(
+            device
+        ), self.train_ar_mask.to(device)
+        self.test_input, self.test_ar_mask = self.test_input.to(
+            device
+        ), self.test_ar_mask.to(device)
+
+        self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1
+
+        if result_dir is not None:
+            self.save_image(
+                self.train_input[:96], result_dir, f"world_train.png", logger
+            )
+
+    def batches(self, split="train", nb_to_use=-1, desc=None):
+        assert split in {"train", "test"}
+        input = self.train_input if split == "train" else self.test_input
+        if nb_to_use > 0:
+            input = input[:nb_to_use]
+        if desc is None:
+            desc = f"epoch-{split}"
+        for batch in tqdm.tqdm(
+            input.split(self.batch_size), dynamic_ncols=True, desc=desc
+        ):
+            yield batch
+
+    def vocabulary_size(self):
+        return self.nb_codes
+
+    def produce_results(
+        self, n_epoch, model, result_dir, logger, deterministic_synthesis, nmax=1000
+    ):
+        def compute_accuracy(input, ar_mask, logger=None):
+            input, ar_mask = input[:nmax], ar_mask[:nmax]
+            result = input.clone() * (1 - ar_mask)
+
+            masked_inplace_autoregression(
+                model,
+                self.batch_size,
+                result,
+                ar_mask,
+                deterministic_synthesis,
+                progress_bar_desc=None,
+                device=self.device,
+            )
+
+            nb_total, nb_correct = (
+                input.size(0),
+                (input == result).long().min(dim=1).values.sum(),
+            )
+
+            return nb_total, nb_correct
+
+        train_nb_total, train_nb_correct = compute_accuracy(
+            self.train_input, self.train_ar_mask
+        )
+
+        logger(
+            f"accuracy_train {n_epoch} nb_total {train_nb_total} nb_correct {train_nb_correct} accuracy {(100.0*train_nb_correct)/train_nb_total:.02f}%"
+        )
+
+        test_nb_total, test_nb_correct = compute_accuracy(
+            self.test_input, self.test_ar_mask, logger
+        )
+
+        logger(
+            f"accuracy_test {n_epoch} nb_total {test_nb_total} nb_correct {test_nb_correct} accuracy {(100.0*test_nb_correct)/test_nb_total:.02f}%"
+        )
+
+        logger(f"main_test_accuracy {n_epoch} {test_nb_correct/test_nb_total}")
+
+        ##############################
+
+        input, ar_mask = self.test_input[:96], self.test_ar_mask[:96]
+        result = input.clone() * (1 - ar_mask)
+
+        masked_inplace_autoregression(
+            model,
+            self.batch_size,
+            result,
+            ar_mask,
+            deterministic_synthesis,
+            progress_bar_desc=None,
+            device=self.device,
+        )
+
+        self.save_image(result, result_dir, f"world_result_{n_epoch:04d}.png", logger)
+
+    def store_new_problems(self, new_problems):
+        nb_current = self.train_input.size(0)
+        nb_new = new_problems.size(0)
+        if nb_new >= nb_current:
+            self.train_input[...] = new_problems[:nb_current]
+        else:
+            nb_kept = nb_current - nb_new
+            self.train_input[:nb_kept] = self.train_input[-nb_kept:].clone()
+            self.train_input[nb_kept:] = new_problems
+
+    def create_new_problems(self, n_epoch, result_dir, logger, nb, model, nb_runs):
+        new_problems = torch.empty(
+            nb, self.height * self.width * 2 + 1, device=self.device, dtype=torch.int64
+        )
+        ar_mask = torch.full(new_problems.size(), 1, device=self.device)
+
+        masked_inplace_autoregression(
+            model,
+            self.batch_size,
+            new_problems,
+            ar_mask,
+            deterministic_synthesis=False,
+            progress_bar_desc="new problems",
+            device=self.device,
+        )
+
+        nb_correct = torch.empty(nb, device=self.device, dtype=torch.int64)
+
+        for n in tqdm.tqdm(
+            range(new_problems.size(0)), dynamic_ncols=True, desc="checking problems"
+        ):
+            result = new_problems[n][None, :].expand(nb_runs, -1).clone()
+            ar_mask = (
+                (torch.arange(result.size(1), device=self.device) > result.size(1) // 2)
+                .long()[None, :]
+                .expand_as(result)
+            )
+
+            masked_inplace_autoregression(
+                model,
+                self.batch_size,
+                result,
+                ar_mask,
+                deterministic_synthesis=False,
+                progress_bar_desc=None,
+                device=self.device,
+            )
+
+            nb_correct[n] = (
+                (new_problems[n][None, :] == result).long().min(dim=1).values.sum()
+            )
+
+        return new_problems, nb_correct