projects / culture.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Merge branch 'dev'
[culture.git] / sky.py
diff --git a/sky.py b/sky.py
index 1164185..cc5bd4f 100755 (executable)
--- a/sky.py
+++ b/sky.py
@@ -5,7 +5,7 @@
 
 # Written by Francois Fleuret <francois@fleuret.org>
 
 
 # Written by Francois Fleuret <francois@fleuret.org>
 
-import math, sys, tqdm, os
+import math, sys, tqdm, os, warnings
 
 import torch, torchvision
 
 
 import torch, torchvision
 
@@ -37,8 +37,6 @@ class Sky(problem.Problem):
     token_background = 0
     first_bird_token = 1
     nb_bird_tokens = colors.size(0) - 1
     token_background = 0
     first_bird_token = 1
     nb_bird_tokens = colors.size(0) - 1
-    token_forward = first_bird_token + nb_bird_tokens
-    token_backward = token_forward + 1
 
     token2char = (
         "_" + "".join([chr(ord("A") + n) for n in range(len(colors) - 1)]) + "><"
 
     token2char = (
         "_" + "".join([chr(ord("A") + n) for n in range(len(colors) - 1)]) + "><"
@@ -52,7 +50,11 @@ class Sky(problem.Problem):
         speed=2,
         nb_iterations=2,
         avoid_collision=True,
         speed=2,
         nb_iterations=2,
         avoid_collision=True,
+        max_nb_cached_chunks=None,
+        chunk_size=None,
+        nb_threads=-1,
     ):
     ):
+        super().__init__(max_nb_cached_chunks, chunk_size, nb_threads)
         self.height = height
         self.width = width
         self.nb_birds = nb_birds
         self.height = height
         self.width = width
         self.nb_birds = nb_birds
@@ -60,9 +62,6 @@ class Sky(problem.Problem):
         self.nb_iterations = nb_iterations
         self.avoid_collision = avoid_collision
 
         self.nb_iterations = nb_iterations
         self.avoid_collision = avoid_collision
 
-    def direction_tokens(self):
-        return self.token_forward, self.token_backward
-
     def generate_frame_sequences(self, nb):
         frame_sequences = []
 
     def generate_frame_sequences(self, nb):
         frame_sequences = []
 
@@ -157,32 +156,8 @@ class Sky(problem.Problem):
 
     ######################################################################
 
 
     ######################################################################
 
-    def generate_token_sequences(self, nb):
-        frame_sequences = self.generate_frame_sequences(nb)
-
-        result = []
-
-        for frame_sequence in frame_sequences:
-            a = []
-            if torch.rand(1) < 0.5:
-                for frame in frame_sequence:
-                    if len(a) > 0:
-                        a.append(torch.tensor([self.token_forward]))
-                    a.append(frame.flatten())
-            else:
-                for frame in reversed(frame_sequence):
-                    if len(a) > 0:
-                        a.append(torch.tensor([self.token_backward]))
-                    a.append(frame.flatten())
-
-            result.append(torch.cat(a, dim=0)[None, :])
-
-        return torch.cat(result, dim=0)
-
-    ######################################################################
-
     def frame2img(self, x, scale=15):
     def frame2img(self, x, scale=15):
-        x = x.reshape(-1, self.height, self.width)
+        x = x.reshape(x.size(0), self.height, -1)
         m = torch.logical_and(
             x >= 0, x < self.first_bird_token + self.nb_bird_tokens
         ).long()
         m = torch.logical_and(
             x >= 0, x < self.first_bird_token + self.nb_bird_tokens
         ).long()
@@ -208,92 +183,140 @@ class Sky(problem.Problem):
 
         return x
 
 
         return x
 
-    def seq2img(self, seq, scale=15):
-        all = [
-            self.frame2img(
-                seq[:, : self.height * self.width].reshape(-1, self.height, self.width),
-                scale,
-            )
-        ]
+    def seq2str(self, seq):
+        result = []
+        for s in seq:
+            result.append("".join([self.token2char[v] for v in s]))
+        return result
 
 
-        separator = torch.full((seq.size(0), 3, self.height * scale - 1, 1), 0)
+    def save_image(
+        self,
+        result_dir,
+        filename,
+        prompts,
+        answers,
+        predicted_prompts=None,
+        predicted_answers=None,
+    ):
+        if predicted_prompts is None:
+            predicted_prompts = 255
+
+        if predicted_answers is None:
+            predicted_answers = 255
 
 
-        t = self.height * self.width
+        def add_frame(x, c, margin, bottom=False):
+            if bottom:
+                h, w, di, dj = x.size(2) + margin, x.size(3), 0, 0
+            else:
+                h, w, di, dj = (
+                    x.size(2) + 2 * margin,
+                    x.size(3) + 2 * margin,
+                    margin,
+                    margin,
+                )
 
 
-        while t < seq.size(1):
-            direction_tokens = seq[:, t]
-            t += 1
+            y = x.new_full((x.size(0), x.size(1), h, w), 0)
 
 
-            direction_images = self.colors[
-                torch.full(
-                    (direction_tokens.size(0), self.height * scale - 1, scale), 0
+            if type(c) is int:
+                y[...] = c
+            else:
+                c = c.long()[:, None]
+                c = (
+                    (c == 1).long() * torch.tensor([0, 255, 0], device=c.device)
+                    + (c == 0).long() * torch.tensor([255, 255, 255], device=c.device)
+                    + (c == -1).long() * torch.tensor([255, 0, 0], device=c.device)
                 )
                 )
-            ].permute(0, 3, 1, 2)
-
-            for n in range(direction_tokens.size(0)):
-                if direction_tokens[n] == self.token_forward:
-                    for k in range(scale):
-                        for l in [0, 1]:
-                            direction_images[
-                                n,
-                                :,
-                                (self.height * scale) // 2 - scale // 2 + k - l,
-                                3 + scale // 2 - abs(k - scale // 2),
-                            ] = 0
-                elif direction_tokens[n] == self.token_backward:
-                    for k in range(scale):
-                        for l in [0, 1]:
-                            direction_images[
-                                n,
-                                :,
-                                (self.height * scale) // 2 - scale // 2 + k - l,
-                                3 + abs(k - scale // 2),
-                            ] = 0
-                else:
-                    for k in range(2, scale - 2):
-                        for l in [0, 1]:
-                            direction_images[
-                                n,
-                                :,
-                                (self.height * scale) // 2 - scale // 2 + k - l,
-                                k,
-                            ] = 0
-                            direction_images[
-                                n,
-                                :,
-                                (self.height * scale) // 2 - scale // 2 + k - l,
-                                scale - 1 - k,
-                            ] = 0
-
-            all += [
-                separator,
-                direction_images,
-                separator,
-                self.frame2img(
-                    seq[:, t : t + self.height * self.width].reshape(
-                        -1, self.height, self.width
-                    ),
-                    scale,
-                ),
-            ]
-
-            t += self.height * self.width
-
-        return torch.cat(all, dim=3)
+                y[...] = c[:, :, None, None]
 
 
-    def seq2str(self, seq):
-        result = []
-        for s in seq:
-            result.append("".join([self.token2char[v] for v in s]))
-        return result
+            y[:, :, di : di + x.size(2), dj : dj + x.size(3)] = x
+
+            return y
+
+        margin = 4
+
+        img_prompts = add_frame(self.frame2img(prompts.to("cpu")), c=0, margin=1)
+        h = img_prompts.size(2)
+        img_answers = add_frame(self.frame2img(answers.to("cpu")), c=0, margin=1)
+
+        img_prompts = add_frame(img_prompts, c=255, margin=margin, bottom=True)
+        img_answers = add_frame(img_answers, c=255, margin=margin, bottom=True)
+
+        img_prompts = add_frame(
+            img_prompts, c=predicted_prompts, margin=margin, bottom=True
+        )
+        img_answers = add_frame(
+            img_answers, c=predicted_answers, margin=margin, bottom=True
+        )
+
+        marker_size = 16
+
+        separator = img_prompts.new_full(
+            (
+                img_prompts.size(0),
+                img_prompts.size(1),
+                img_prompts.size(2),
+                marker_size,
+            ),
+            255,
+        )
+
+        separator[:, :, 0] = 0
+        separator[:, :, h - 1] = 0
+
+        for k in range(1, 2 * marker_size - 8):
+            i = k - (marker_size - 4)
+            j = marker_size - 5 - abs(i)
+            separator[:, :, h // 2 - 1 + i, 2 + j] = 0
+            separator[:, :, h // 2 - 1 + i + 1, 2 + j] = 0
+
+        img = torch.cat([img_prompts, separator, img_answers], dim=3)
 
 
-    def save_image(self, input, result_dir, filename):
-        img = self.seq2img(input.to("cpu"))
         image_name = os.path.join(result_dir, filename)
         image_name = os.path.join(result_dir, filename)
-        torchvision.utils.save_image(img.float() / 255.0, image_name, nrow=6, padding=4)
+        torchvision.utils.save_image(
+            img.float() / 255.0, image_name, nrow=6, padding=margin * 4, pad_value=1.0
+        )
+
+    ######################################################################
+
+    def nb_token_values(self):
+        return len(self.colors)
+
+    def generate_prompts_and_answers(self, nb):
+        frame_sequences = self.generate_frame_sequences(nb)
+        frame_sequences = torch.cat([x[None] for x in frame_sequences], dim=0)
+
+        prompts = frame_sequences[:, : frame_sequences.size(1) // 2].flatten(1)
+
+        answers = frame_sequences[:, frame_sequences.size(1) // 2 :].flatten(1)
+
+        # warnings.warn("dirty test with longer answer", RuntimeWarning)
+        # answers = torch.cat(
+        # [
+        # frame_sequences[:, frame_sequences.size(1) // 2 :],
+        # frame_sequences[:, frame_sequences.size(1) // 2 :],
+        # ],
+        # dim=3,
+        # ).flatten(1)
 
 
-    def save_quizzes(self, input, result_dir, filename_prefix):
-        self.save_image(input, result_dir, filename_prefix + ".png")
+        return prompts, answers
+
+    def save_quiz_illustrations(
+        self,
+        result_dir,
+        filename_prefix,
+        prompts,
+        answers,
+        predicted_prompts=None,
+        predicted_answers=None,
+    ):
+        self.save_image(
+            result_dir,
+            filename_prefix + ".png",
+            prompts,
+            answers,
+            predicted_prompts,
+            predicted_answers,
+        )
 
 
 ######################################################################
 
 
 ######################################################################
@@ -301,12 +324,21 @@ class Sky(problem.Problem):
 if __name__ == "__main__":
     import time
 
 if __name__ == "__main__":
     import time
 
-    sky = Sky(height=6, width=8, speed=4, nb_iterations=2)
+    sky = Sky(height=6, width=8, speed=1, nb_iterations=4)
 
 
-    start_time = time.perf_counter()
-    token_sequences = sky.generate_token_sequences(nb=64)
-    delay = time.perf_counter() - start_time
-    print(f"{token_sequences.size(0)/delay:02f} seq/s")
+    prompts, answers = sky.generate_prompts_and_answers(4)
+
+    predicted_prompts = torch.randint(3, (prompts.size(0),)) - 1
+    predicted_answers = torch.randint(3, (prompts.size(0),)) - 1
+
+    sky.save_quiz_illustrations(
+        "/tmp", "test", prompts, answers, predicted_prompts, predicted_answers
+    )
+
+    # start_time = time.perf_counter()
+    # token_sequences = sky.generate_token_sequences(nb=64)
+    # delay = time.perf_counter() - start_time
+    # print(f"{token_sequences.size(0)/delay:02f} seq/s")
 
     # print(sky.seq2str(seq[:4]))
 
 
     # print(sky.seq2str(seq[:4]))
 
@@ -324,9 +356,9 @@ if __name__ == "__main__":
     # seq = (1 - m) * seq + m * 23
 
     # print(seq.size())
     # seq = (1 - m) * seq + m * 23
 
     # print(seq.size())
-    img = sky.seq2img(token_sequences)
+    # img = sky.seq2img(token_sequences)
     # print(img.size())
 
     # print(img.size())
 
-    torchvision.utils.save_image(
-        img.float() / 255.0, "/tmp/world.png", nrow=6, padding=6, pad_value=0
-    )
+    # torchvision.utils.save_image(
+    # img.float() / 255.0, "/tmp/world.png", nrow=6, padding=6, pad_value=0
+    # )
An attempt at having GPTs general their own quizzes.