X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=blobdiff_plain;f=tasks.py;h=b967465de6c23a94b9ee1af8a672337167903ac8;hb=2a9d74f8b24441dc56505f9bdce8a2974e2552cb;hp=2c883339b0ac15610f0a425891a1e1e837b1a325;hpb=32ca97e128baa0392684fd5a84632651178b3d89;p=culture.git

diff --git a/tasks.py b/tasks.py
index 2c88333..b967465 100755
--- a/tasks.py
+++ b/tasks.py
@@ -22,6 +22,7 @@ def masked_inplace_autoregression(
     batch_size,
     input,
     ar_mask,
+    temperature,
     deterministic_synthesis,
     forbidden_tokens=None,
     logit_biases=None,
@@ -44,17 +45,22 @@ def masked_inplace_autoregression(
         t = model.training
         model.eval()
 
+        sum_logits = 0
+
         for input, ar_mask in batches:
-            model.masked_inplace_autoregression(
-                input,
-                ar_mask,
-                deterministic_synthesis,
-                forbidden_tokens,
-                logit_biases,
+            sum_logits += model.masked_inplace_autoregression(
+                input=input,
+                ar_mask=ar_mask,
+                temperature=temperature,
+                deterministic_synthesis=deterministic_synthesis,
+                forbidden_tokens=forbidden_tokens,
+                forced_biases=logit_biases,
             )
 
         model.train(t)
 
+        return sum_logits
+
 
 ######################################################################
 
@@ -79,7 +85,7 @@ import world
 
 class World(Task):
     def save_image(self, input, result_dir, filename, logger):
-        img = world.sample2img(input.to("cpu"), self.height, self.width)
+        img = world.seq2img(input.to("cpu"), self.height, self.width)
         image_name = os.path.join(result_dir, filename)
         torchvision.utils.save_image(img.float() / 255.0, image_name, nrow=6, padding=4)
         logger(f"wrote {image_name}")
@@ -167,11 +173,12 @@ class World(Task):
             result = input.clone() * (1 - ar_mask)
 
             masked_inplace_autoregression(
-                model,
-                self.batch_size,
-                result,
-                ar_mask,
-                deterministic_synthesis,
+                model=model,
+                batch_size=self.batch_size,
+                input=result,
+                ar_mask=ar_mask,
+                temperature=1.0,
+                deterministic_synthesis=deterministic_synthesis,
                 progress_bar_desc=None,
                 device=self.device,
             )
@@ -205,11 +212,12 @@ class World(Task):
         result = input.clone() * (1 - ar_mask)
 
         masked_inplace_autoregression(
-            model,
-            self.batch_size,
-            result,
-            ar_mask,
-            deterministic_synthesis,
+            model=model,
+            batch_size=self.batch_size,
+            input=result,
+            ar_mask=ar_mask,
+            temperature=1.0,
+            deterministic_synthesis=deterministic_synthesis,
             progress_bar_desc=None,
             device=self.device,
         )
@@ -245,6 +253,7 @@ class World(Task):
         nb,
         model,
         other_models,
+        desired_average_logits=None,
     ):
         ###############################################################
         # Generate quizzes with model
@@ -254,16 +263,35 @@ class World(Task):
         )
         ar_mask = torch.full(quizzes.size(), 1, device=self.device)
 
-        masked_inplace_autoregression(
-            model,
-            self.batch_size,
-            quizzes,
-            ar_mask,
+        sum_logits = masked_inplace_autoregression(
+            model=model,
+            batch_size=self.batch_size,
+            input=quizzes,
+            ar_mask=ar_mask,
+            temperature=1.0,
             deterministic_synthesis=False,
             progress_bar_desc="creating quizzes",
             device=self.device,
         )
 
+        average_logits = sum_logits / quizzes.numel()
+
+        # It's a bit brutal to do it twice, we should probably have a
+        # moving average and apply it right away
+
+        if desired_average_logits is not None:
+            temperature = average_logits / desired_average_logits
+            masked_inplace_autoregression(
+                model=model,
+                batch_size=self.batch_size,
+                input=quizzes,
+                ar_mask=ar_mask,
+                temperature=temperature,
+                deterministic_synthesis=False,
+                progress_bar_desc="creating quizzes",
+                device=self.device,
+            )
+
         ###############################################################
         # Create the reverse quizzes
 
@@ -288,10 +316,11 @@ class World(Task):
             result = quizzes.clone()
 
             masked_inplace_autoregression(
-                m,
-                self.batch_size,
-                result,
-                ar_mask,
+                model=m,
+                batch_size=self.batch_size,
+                input=result,
+                ar_mask=ar_mask,
+                temperature=1.0,
                 deterministic_synthesis=True,
                 progress_bar_desc="solving quizzes",
                 device=self.device,
@@ -302,10 +331,11 @@ class World(Task):
             reverse_result = reverse_quizzes.clone()
 
             masked_inplace_autoregression(
-                m,
-                self.batch_size,
-                reverse_result,
-                ar_mask,
+                model=m,
+                batch_size=self.batch_size,
+                input=reverse_result,
+                ar_mask=ar_mask,
+                temperature=1.0,
                 deterministic_synthesis=True,
                 progress_bar_desc="solving reversed quizzes",
                 device=self.device,
@@ -324,4 +354,4 @@ class World(Task):
             for k in nb_correct:
                 f.write(f"{k}\n")
 
-        return quizzes, nb_correct.sum(dim=0)
+        return quizzes, nb_correct.sum(dim=0), average_logits