Update.

[culture.git] / quizz_machine.py

diff --git a/quizz_machine.py b/quizz_machine.py
index 6cad6a1..84bb558 100755 (executable)
--- a/quizz_machine.py
+++ b/quizz_machine.py
@@ -12,6 +12,7 @@ import torch, torchvision
 from torch import nn
 from torch.nn import functional as F
 
+import mygpt
 from mygpt import BracketedSequence
 
 ######################################################################
@@ -20,7 +21,7 @@ from mygpt import BracketedSequence
 class Gang(nn.Module):
     def __init__(self, models, nb_models_for_generation, mode="groupthink"):
         super().__init__()
-        self.models = models
+        self.models = nn.ModuleList(models)
         self.nb_models_for_generation = nb_models_for_generation
         self.mode = mode
 
@@ -383,58 +384,39 @@ class QuizzMachine:
         ar_mask_solve = 1 - ar_mask_prompt
         seq_logproba = torch.empty(ar_mask_prompt.size(0), device=self.device)
 
-        # bracketing of the temperature to get the target logproba if
-        # min_ave_seq_logproba is not None
+        warnings.warn("noise injection", RuntimeWarning)
+        temperature = 1
+        noise_std = torch.rand(1).item()
+        self.logger(f"{noise_std=}")
+        mygpt.set_noise_injection(model_for_generation, noise_std)
 
-        temperature = 2
-        d_temperature = 1 / 3
-
-        while True:
-            seq_logproba[...] = 0
-
-            masked_inplace_autoregression(
-                model=model_for_generation,
-                batch_size=self.batch_size,
-                input=c_quizzes,
-                ar_mask=ar_mask_prompt,
-                seq_logproba=seq_logproba,
-                temperature=temperature,
-                deterministic_synthesis=False,
-                # progress_bar_desc="sampling c_quizzes",
-                device=self.device,
-            )
+        masked_inplace_autoregression(
+            model=model_for_generation,
+            batch_size=self.batch_size,
+            input=c_quizzes,
+            ar_mask=ar_mask_prompt,
+            seq_logproba=seq_logproba,
+            temperature=temperature,
+            deterministic_synthesis=False,
+            # progress_bar_desc="sampling c_quizzes",
+            device=self.device,
+        )
 
-            ave_seq_logproba = seq_logproba.mean()
+        ave_seq_logproba = seq_logproba.mean()
 
-            masked_inplace_autoregression(
-                model=model_for_generation,
-                batch_size=self.batch_size,
-                input=c_quizzes,
-                ar_mask=ar_mask_solve,
-                seq_logproba=seq_logproba,
-                temperature=temperature,
-                deterministic_synthesis=True,
-                # progress_bar_desc="sampling c_quizzes",
-                device=self.device,
-            )
+        masked_inplace_autoregression(
+            model=model_for_generation,
+            batch_size=self.batch_size,
+            input=c_quizzes,
+            ar_mask=ar_mask_solve,
+            seq_logproba=seq_logproba,
+            temperature=temperature,
+            deterministic_synthesis=True,
+            # progress_bar_desc="sampling c_quizzes",
+            device=self.device,
+        )
 
-            # If we do not have target logprobs, get out now
-            if min_ave_seq_logproba is None:
-                break
-
-            # Oh man that's ugly
-            if ave_seq_logproba < min_ave_seq_logproba:
-                if d_temperature > 0:
-                    d_temperature *= -1 / 3
-                temperature += d_temperature
-            elif ave_seq_logproba > min_ave_seq_logproba * 0.99:
-                if d_temperature < 0:
-                    d_temperature *= -1 / 3
-                temperature += d_temperature
-            else:
-                break
-
-            self.logger(f"changing temperature to {temperature}")
+        mygpt.set_noise_injection(model_for_generation, 0.0)
 
         return c_quizzes, seq_logproba.mean()