X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=sky.py;h=6ef8a3af2184777c223dfd1803647a49bd3dd54d;hb=167c56ace610c3b975c702203bb7c7ddf74930ae;hp=cb25ea0ec335fc1823d2f4d7d044ed7908184a0f;hpb=c8979c695ad584c54d605b8f183e5d2e99f2d1cc;p=culture.git

diff --git a/sky.py b/sky.py
index cb25ea0..6ef8a3a 100755
--- a/sky.py
+++ b/sky.py
@@ -5,7 +5,7 @@
 
 # Written by Francois Fleuret <francois@fleuret.org>
 
-import math, sys, tqdm, os
+import math, sys, tqdm, os, warnings
 
 import torch, torchvision
 
@@ -14,19 +14,10 @@ from torch.nn import functional as F
 
 ######################################################################
 
+import problem
 
-class Problem:
-    def generate_seq(self, nb_train_samples):
-        pass
 
-    def save_quizzes(self, input, result_dir, filename_prefix, logger):
-        pass
-
-    def direction_tokens(self):
-        pass
-
-
-class Sky:
+class Sky(problem.Problem):
     colors = torch.tensor(
         [
             [255, 255, 255],
@@ -46,91 +37,101 @@ class Sky:
     token_background = 0
     first_bird_token = 1
     nb_bird_tokens = colors.size(0) - 1
-    token_forward = first_bird_token + nb_bird_tokens
-    token_backward = token_forward + 1
 
     token2char = (
         "_" + "".join([chr(ord("A") + n) for n in range(len(colors) - 1)]) + "><"
     )
 
-    def __init__(self, height=6, width=8, nb_birds=3, nb_iterations=2):
+    def __init__(
+        self,
+        height=6,
+        width=8,
+        nb_birds=3,
+        speed=2,
+        nb_iterations=2,
+        avoid_collision=True,
+    ):
         self.height = height
         self.width = width
         self.nb_birds = nb_birds
+        self.speed = speed
         self.nb_iterations = nb_iterations
+        self.avoid_collision = avoid_collision
 
-    def direction_tokens(self):
-        return self.token_forward, self.token_backward
-
-    def generate_seq(self, nb, return_iterations=False):
-        pairs = []
-        kept_iterations = []
+    def generate_frame_sequences(self, nb):
+        frame_sequences = []
 
         for _ in tqdm.tqdm(range(nb), dynamic_ncols=True, desc="world generation"):
-            while True:
-                iterations = []
-
-                f_start = torch.zeros(self.height, self.width, dtype=torch.int64)
+            i, j, vi, vj = (
+                torch.empty(self.nb_birds, dtype=torch.int64),
+                torch.empty(self.nb_birds, dtype=torch.int64),
+                torch.empty(self.nb_birds, dtype=torch.int64),
+                torch.empty(self.nb_birds, dtype=torch.int64),
+            )
 
-                i, j, vi, vj = (
-                    torch.empty(self.nb_birds, dtype=torch.int64),
-                    torch.empty(self.nb_birds, dtype=torch.int64),
-                    torch.empty(self.nb_birds, dtype=torch.int64),
-                    torch.empty(self.nb_birds, dtype=torch.int64),
-                )
+            def collision_okay():
+                if not self.avoid_collision:
+                    return True
 
-                col = (
-                    torch.randperm(self.colors.size(0) - 1)[: self.nb_birds]
-                    .sort()
-                    .values
-                    + 1
-                )
+                count = torch.zeros(self.height, self.width, dtype=torch.int64)
 
                 for n in range(self.nb_birds):
-                    c = col[n]
-
-                    while True:
-                        i[n], j[n] = (
-                            torch.randint(self.height, (1,))[0],
-                            torch.randint(self.width, (1,))[0],
-                        )
-                        vm = torch.randint(4, (1,))[0]
-                        vi[n], vj[n] = (vm % 2) * 2 - 1, (vm // 2) * 2 - 1
-                        if (
-                            i[n] - vi[n] >= 0
-                            and i[n] - vi[n] < self.height
-                            and j[n] - vj[n] >= 0
-                            and j[n] - vj[n] < self.width
-                            and f_start[i[n], j[n]] == 0
-                            and f_start[i[n] - vi[n], j[n]] == 0
-                            and f_start[i[n], j[n] - vj[n]] == 0
-                        ):
-                            break
+                    count[i[n], j[n]] += 1
+                    count[i[n] - vi[n], j[n]] += 1
+                    count[i[n], j[n] - vj[n]] += 1
+
+                return count.max() <= 1
 
-                    f_start[i[n], j[n]] = c
-                    f_start[i[n] - vi[n], j[n]] = c
-                    f_start[i[n], j[n] - vj[n]] = c
+            col = (
+                torch.randperm(self.colors.size(0) - 1)[: self.nb_birds].sort().values
+                + 1
+            )
 
-                f_end = f_start.clone()
+            while True:
+                while True:
+                    for n in range(self.nb_birds):
+                        while True:
+                            i[n] = torch.randint(self.height, (1,))
+                            j[n] = torch.randint(self.width, (1,))
+                            vm = torch.randint(4, (1,))
+                            vi[n], vj[n] = (vm % 2) * 2 - 1, (vm // 2) * 2 - 1
+                            if (
+                                i[n] - vi[n] >= 0
+                                and i[n] - vi[n] < self.height
+                                and j[n] - vj[n] >= 0
+                                and j[n] - vj[n] < self.width
+                            ):
+                                break
+
+                    if collision_okay():
+                        break
+
+                result = torch.zeros(
+                    self.nb_iterations * self.speed,
+                    self.height,
+                    self.width,
+                    dtype=torch.int64,
+                )
+
+                fine = torch.empty(self.nb_iterations * self.speed)
+
+                t_to_keep = (
+                    torch.arange(self.nb_iterations, device=result.device) * self.speed
+                )
 
-                for l in range(self.nb_iterations):
-                    iterations.append(f_end.clone())
-                    f_end[...] = 0
-                    nb_collisions = 0
+                for l in range(self.nb_iterations * self.speed):
+                    fine[l] = collision_okay()
                     for n in range(self.nb_birds):
                         c = col[n]
-
-                        pi, pj, pvi, pvj = (
-                            i[n].item(),
-                            j[n].item(),
-                            vi[n].item(),
-                            vj[n].item(),
-                        )
+                        result[l, i[n], j[n]] = c
+                        result[l, i[n] - vi[n], j[n]] = c
+                        result[l, i[n], j[n] - vj[n]] = c
 
                         if (i[n] == 0 and vi[n] == -1) or (
                             i[n] == self.height - 1 and vi[n] == 1
                         ):
                             vi[n] = -vi[n]
+
                         if (j[n] == 0 and vj[n] == -1) or (
                             j[n] == self.width - 1 and vj[n] == 1
                         ):
@@ -139,227 +140,177 @@ class Sky:
                         i[n] += vi[n]
                         j[n] += vj[n]
 
-                        if not (
-                            f_end[i[n], j[n]] == 0
-                            and f_end[i[n] - vi[n], j[n]] == 0
-                            and f_end[i[n], j[n] - vj[n]] == 0
-                        ):
-                            nb_collisions += 1
-
-                        f_end[i[n], j[n]] = c
-                        f_end[i[n] - vi[n], j[n]] = c
-                        f_end[i[n], j[n] - vj[n]] = c
-
-                iterations.append(f_end.clone())
+                result = result[t_to_keep]
+                fine = fine[t_to_keep]
 
-                if nb_collisions == 0:
+                if fine[-1]:
                     break
 
-            kept_iterations.append(iterations)
-            pairs.append((f_start, f_end))
+            frame_sequences.append(result)
 
-        result = []
-        for p in pairs:
-            if torch.rand(1) < 0.5:
-                result.append(
-                    torch.cat(
-                        [
-                            p[0].flatten(),
-                            torch.tensor([self.token_forward]),
-                            p[1].flatten(),
-                        ],
-                        dim=0,
-                    )[None, :]
-                )
-            else:
-                result.append(
-                    torch.cat(
-                        [
-                            p[1].flatten(),
-                            torch.tensor([self.token_backward]),
-                            p[0].flatten(),
-                        ],
-                        dim=0,
-                    )[None, :]
-                )
-
-        if return_iterations:
-            # iterations = torch.cat([ torch.cat([ x[None, None] for x in l], dim = 1) for l in kept_iterations ], dim=0)
-            return torch.cat(result, dim=0), kept_iterations
-        else:
-            return torch.cat(result, dim=0)
+        return frame_sequences
 
     ######################################################################
 
-    def generate_seq_old(
-        self,
-        nb,
-    ):
-        pairs = []
-
-        for n in tqdm.tqdm(range(nb), dynamic_ncols=True, desc="world generation"):
-            f_start = torch.zeros(self.height, self.width, dtype=torch.int64)
-            f_end = torch.zeros(self.height, self.width, dtype=torch.int64)
-            n = torch.arange(f_start.size(0))
-
-            for c in (
-                (torch.randperm(self.nb_bird_tokens) + self.first_bird_token)[
-                    : self.nb_birds
-                ]
-                .sort()
-                .values
-            ):
-                i, j = (
-                    torch.randint(self.height - 2, (1,))[0] + 1,
-                    torch.randint(self.width - 2, (1,))[0] + 1,
-                )
-                vm = torch.randint(4, (1,))[0]
-                vi, vj = (vm // 2) * (2 * (vm % 2) - 1), (1 - vm // 2) * (
-                    2 * (vm % 2) - 1
-                )
-
-                f_start[i, j] = c
-                f_start[i - vi, j - vj] = c
-                f_start[i + vj, j - vi] = c
-                f_start[i - vj, j + vi] = c
-
-                for l in range(self.nb_iterations):
-                    i += vi
-                    j += vj
-                    if i < 0 or i >= self.height or j < 0 or j >= self.width:
-                        i -= vi
-                        j -= vj
-                        vi, vj = -vi, -vj
-                        i += vi
-                        j += vj
-
-                f_end[i, j] = c
-                f_end[i - vi, j - vj] = c
-                f_end[i + vj, j - vi] = c
-                f_end[i - vj, j + vi] = c
-
-            pairs.append((f_start, f_end))
-
-        result = []
-        for p in pairs:
-            if torch.rand(1) < 0.5:
-                result.append(
-                    torch.cat(
-                        [
-                            p[0].flatten(),
-                            torch.tensor([self.token_forward]),
-                            p[1].flatten(),
-                        ],
-                        dim=0,
-                    )[None, :]
-                )
-            else:
-                result.append(
-                    torch.cat(
-                        [
-                            p[1].flatten(),
-                            torch.tensor([self.token_backward]),
-                            p[0].flatten(),
-                        ],
-                        dim=0,
-                    )[None, :]
-                )
-
-        return torch.cat(result, dim=0)
-
-    def frame2img(self, x, upscale=15):
-        x = x.reshape(-1, self.height, self.width)
+    def frame2img(self, x, scale=15):
+        x = x.reshape(x.size(0), self.height, -1)
         m = torch.logical_and(
             x >= 0, x < self.first_bird_token + self.nb_bird_tokens
         ).long()
         x = self.colors[x * m].permute(0, 3, 1, 2)
         s = x.shape
-        x = x[:, :, :, None, :, None].expand(-1, -1, -1, upscale, -1, upscale)
-        x = x.reshape(s[0], s[1], s[2] * upscale, s[3] * upscale)
+        x = x[:, :, :, None, :, None].expand(-1, -1, -1, scale, -1, scale)
+        x = x.reshape(s[0], s[1], s[2] * scale, s[3] * scale)
 
-        x[:, :, :, torch.arange(0, x.size(3), upscale)] = 0
-        x[:, :, torch.arange(0, x.size(2), upscale), :] = 0
+        x[:, :, :, torch.arange(0, x.size(3), scale)] = 0
+        x[:, :, torch.arange(0, x.size(2), scale), :] = 0
         x = x[:, :, 1:, 1:]
 
         for n in range(m.size(0)):
             for i in range(m.size(1)):
                 for j in range(m.size(2)):
                     if m[n, i, j] == 0:
-                        for k in range(2, upscale - 2):
-                            x[n, :, i * upscale + k, j * upscale + k] = 0
-                            x[n, :, i * upscale + upscale - 1 - k, j * upscale + k] = 0
+                        for k in range(2, scale - 2):
+                            for l in [0, 1]:
+                                x[n, :, i * scale + k, j * scale + k - l] = 0
+                                x[
+                                    n, :, i * scale + scale - 1 - k, j * scale + k - l
+                                ] = 0
 
         return x
 
-    def seq2img(self, seq, upscale=15):
-        f_first = seq[:, : self.height * self.width].reshape(
-            -1, self.height, self.width
-        )
-        f_second = seq[:, self.height * self.width + 1 :].reshape(
-            -1, self.height, self.width
-        )
-        direction = seq[:, self.height * self.width]
-
-        direction_symbol = torch.full(
-            (direction.size(0), self.height * upscale - 1, upscale), 0
-        )
-        direction_symbol = self.colors[direction_symbol].permute(0, 3, 1, 2)
-        separator = torch.full((direction.size(0), 3, self.height * upscale - 1, 1), 0)
-
-        for n in range(direction_symbol.size(0)):
-            if direction[n] == self.token_forward:
-                for k in range(upscale):
-                    direction_symbol[
-                        n,
-                        :,
-                        (self.height * upscale) // 2 - upscale // 2 + k,
-                        3 + upscale // 2 - abs(k - upscale // 2),
-                    ] = 0
-            elif direction[n] == self.token_backward:
-                for k in range(upscale):
-                    direction_symbol[
-                        n,
-                        :,
-                        (self.height * upscale) // 2 - upscale // 2 + k,
-                        3 + abs(k - upscale // 2),
-                    ] = 0
-            else:
-                for k in range(2, upscale - 2):
-                    direction_symbol[
-                        n, :, (self.height * upscale) // 2 - upscale // 2 + k, k
-                    ] = 0
-                    direction_symbol[
-                        n,
-                        :,
-                        (self.height * upscale) // 2 - upscale // 2 + k,
-                        upscale - 1 - k,
-                    ] = 0
-
-        return torch.cat(
-            [
-                self.frame2img(f_first, upscale),
-                separator,
-                direction_symbol,
-                separator,
-                self.frame2img(f_second, upscale),
-            ],
-            dim=3,
-        )
-
     def seq2str(self, seq):
         result = []
         for s in seq:
             result.append("".join([self.token2char[v] for v in s]))
         return result
 
-    def save_image(self, input, result_dir, filename, logger):
-        img = self.seq2img(input.to("cpu"))
+    def save_image(
+        self,
+        result_dir,
+        filename,
+        prompts,
+        answers,
+        predicted_prompts=None,
+        predicted_answers=None,
+    ):
+        if predicted_prompts is None:
+            predicted_prompts = 255
+
+        if predicted_answers is None:
+            predicted_answers = 255
+
+        def add_frame(x, c, margin, bottom=False):
+            if bottom:
+                h, w, di, dj = x.size(2) + margin, x.size(3), 0, 0
+            else:
+                h, w, di, dj = (
+                    x.size(2) + 2 * margin,
+                    x.size(3) + 2 * margin,
+                    margin,
+                    margin,
+                )
+
+            y = x.new_full((x.size(0), x.size(1), h, w), 0)
+
+            if type(c) is int:
+                y[...] = c
+            else:
+                c = c.long()[:, None]
+                c = c * torch.tensor([0, 0, 0], device=c.device) + (
+                    1 - c
+                ) * torch.tensor([255, 255, 255], device=c.device)
+                y[...] = c[:, :, None, None]
+
+            y[:, :, di : di + x.size(2), dj : dj + x.size(3)] = x
+
+            return y
+
+        margin = 4
+
+        img_prompts = add_frame(self.frame2img(prompts.to("cpu")), c=0, margin=1)
+        h = img_prompts.size(2)
+        img_answers = add_frame(self.frame2img(answers.to("cpu")), c=0, margin=1)
+
+        img_prompts = add_frame(img_prompts, c=255, margin=margin, bottom=True)
+        img_answers = add_frame(img_answers, c=255, margin=margin, bottom=True)
+
+        img_prompts = add_frame(
+            img_prompts, c=predicted_prompts, margin=margin, bottom=True
+        )
+        img_answers = add_frame(
+            img_answers, c=predicted_answers, margin=margin, bottom=True
+        )
+
+        marker_size = 16
+
+        separator = img_prompts.new_full(
+            (
+                img_prompts.size(0),
+                img_prompts.size(1),
+                img_prompts.size(2),
+                marker_size,
+            ),
+            255,
+        )
+
+        separator[:, :, 0] = 0
+        separator[:, :, h - 1] = 0
+
+        for k in range(1, 2 * marker_size - 8):
+            i = k - (marker_size - 4)
+            j = marker_size - 5 - abs(i)
+            separator[:, :, h // 2 - 1 + i, 2 + j] = 0
+            separator[:, :, h // 2 - 1 + i + 1, 2 + j] = 0
+
+        img = torch.cat([img_prompts, separator, img_answers], dim=3)
+
         image_name = os.path.join(result_dir, filename)
-        torchvision.utils.save_image(img.float() / 255.0, image_name, nrow=6, padding=4)
-        logger(f"wrote {image_name}")
+        torchvision.utils.save_image(
+            img.float() / 255.0, image_name, nrow=6, padding=margin * 4, pad_value=1.0
+        )
+
+    ######################################################################
+
+    def nb_token_values(self):
+        return len(self.colors)
+
+    def generate_prompts_and_answers(self, nb):
+        frame_sequences = self.generate_frame_sequences(nb)
+        frame_sequences = torch.cat([x[None] for x in frame_sequences], dim=0)
 
-    def save_quizzes(self, input, result_dir, filename_prefix, logger):
-        self.save_image(input, result_dir, filename_prefix + ".png", logger)
+        prompts = frame_sequences[:, : frame_sequences.size(1) // 2].flatten(1)
+
+        answers = frame_sequences[:, frame_sequences.size(1) // 2 :].flatten(1)
+
+        # warnings.warn("dirty test with longer answer", RuntimeWarning)
+        # answers = torch.cat(
+        # [
+        # frame_sequences[:, frame_sequences.size(1) // 2 :],
+        # frame_sequences[:, frame_sequences.size(1) // 2 :],
+        # ],
+        # dim=3,
+        # ).flatten(1)
+
+        return prompts, answers
+
+    def save_quizzes(
+        self,
+        result_dir,
+        filename_prefix,
+        prompts,
+        answers,
+        predicted_prompts=None,
+        predicted_answers=None,
+    ):
+        self.save_image(
+            result_dir,
+            filename_prefix + ".png",
+            prompts,
+            answers,
+            predicted_prompts,
+            predicted_answers,
+        )
 
 
 ######################################################################
@@ -367,31 +318,41 @@ class Sky:
 if __name__ == "__main__":
     import time
 
-    sky = Sky(height=6, width=8, nb_iterations=100)
+    sky = Sky(height=6, width=8, speed=1, nb_iterations=4)
 
-    start_time = time.perf_counter()
-    seq, it = sky.generate_seq(nb=64, return_iterations=True)
-    delay = time.perf_counter() - start_time
-    print(f"{seq.size(0)/delay:02f} samples/s")
+    prompts, answers = sky.generate_prompts_and_answers(4)
 
-    print(sky.seq2str(seq[:4]))
+    predicted_prompts = torch.rand(prompts.size(0)) < 0.5
+    predicted_answers = torch.rand(answers.size(0)) < 0.5
 
-    for t in range(len(it[0])):
-        img = torch.cat([sky.frame2img(f[t]) for f in it], dim=0)
-        torchvision.utils.save_image(
-            img.float() / 255.0,
-            f"/tmp/frame_{t:03d}.png",
-            nrow=8,
-            padding=6,
-            pad_value=0,
-        )
+    sky.save_quizzes(
+        "/tmp", "test", prompts, answers, predicted_prompts, predicted_answers
+    )
+
+    # start_time = time.perf_counter()
+    # token_sequences = sky.generate_token_sequences(nb=64)
+    # delay = time.perf_counter() - start_time
+    # print(f"{token_sequences.size(0)/delay:02f} seq/s")
+
+    # print(sky.seq2str(seq[:4]))
+
+    # for t in range(len(it[0])):
+    # img = torch.cat([sky.frame2img(f[t]) for f in it], dim=0)
+    # torchvision.utils.save_image(
+    # img.float() / 255.0,
+    # f"/tmp/frame_{t:03d}.png",
+    # nrow=8,
+    # padding=6,
+    # pad_value=0,
+    # )
 
     # m = (torch.rand(seq.size()) < 0.05).long()
     # seq = (1 - m) * seq + m * 23
 
-    img = sky.seq2img(seq)
-    print(img.size())
+    # print(seq.size())
+    # img = sky.seq2img(token_sequences)
+    # print(img.size())
 
-    torchvision.utils.save_image(
-        img.float() / 255.0, "/tmp/world.png", nrow=6, padding=6, pad_value=0
-    )
+    # torchvision.utils.save_image(
+    # img.float() / 255.0, "/tmp/world.png", nrow=6, padding=6, pad_value=0
+    # )