From: Francois Fleuret <francois@fleuret.org>
Date: Thu, 9 Sep 2010 06:19:10 +0000 (+0200)
Subject: Added the non-disclosed problems.
X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?a=commitdiff_plain;h=dfc7d0cd9b185a982b9f83d6d862824a62000aea;p=svrt.git

Added the non-disclosed problems.
---

diff --git a/Makefile b/Makefile
index 4ed1b40..ec7135d 100644
--- a/Makefile
+++ b/Makefile
@@ -49,16 +49,26 @@ vision_test: misc.o rgb_image.o jpeg_misc.o fusion_sort.o global.o param_parser.
 	vision_problem_1.o \
 	vision_problem_2.o \
 	vision_problem_3.o \
+	vision_problem_4.o \
 	vision_problem_5.o \
 	vision_problem_6.o \
+	vision_problem_7.o \
 	vision_problem_8.o \
+	vision_problem_9.o \
+	vision_problem_10.o \
 	vision_problem_11.o \
 	vision_problem_12.o \
 	vision_problem_13.o \
+	vision_problem_14.o \
+	vision_problem_15.o \
+	vision_problem_16.o \
 	vision_problem_17.o \
 	vision_problem_18.o \
+	vision_problem_19.o \
 	vision_problem_20.o \
 	vision_problem_21.o \
+	vision_problem_22.o \
+	vision_problem_23.o \
 	classifier_reader.o \
         classifier.o naive_bayesian_classifier.o \
 	stump.o boosted_classifier.o \
@@ -69,8 +79,8 @@ vision_test: misc.o rgb_image.o jpeg_misc.o fusion_sort.o global.o param_parser.
 Makefile.depend: *.h *.cc Makefile
 	$(CC) -M *.cc > Makefile.depend
 
-svrt.tgz:
-	cd .. ; tar zcvf /tmp/svrt.tgz svrt/*.{cc,h,sh} svrt/Makefile svrt/README.txt svrt/gpl-3.0.txt
+svrt_full.tgz:
+	cd .. ; tar zcvf /tmp/svrt.tgz svrt_full/*.{cc,h,sh} svrt_full/Makefile svrt_full/README.txt svrt_full/gpl-3.0.txt
 
 clean:
 	\rm -f vision_test *.o Makefile.depend TAGS
diff --git a/README.txt b/README.txt
index eab2d84..23aa7c4 100644
--- a/README.txt
+++ b/README.txt
@@ -11,4 +11,4 @@ problems to generate sample images and test a Boosted classifier. If
 no argument is given, it will run on all the problems by default.
 
 --
-Francois Fleuret, Nov 2009.
+Francois Fleuret, Sep 2010.
diff --git a/doit.sh b/doit.sh
index a6bb4ae..958cbe8 100755
--- a/doit.sh
+++ b/doit.sh
@@ -26,7 +26,7 @@ nb_samples_for_training=1000
 
 problem_list=$*
 
-[[ ${problem_list} ]] || problem_list="1 2 3 5 6 8 11 12 13 17 18 20 21"
+[[ ${problem_list} ]] || problem_list=$(echo {1..23})
 
 set -e
 
diff --git a/vision_problem_10.cc b/vision_problem_10.cc
new file mode 100644
index 0000000..00fdec3
--- /dev/null
+++ b/vision_problem_10.cc
@@ -0,0 +1,54 @@
+
+////////////////////////////////////////////////////////////////////
+// START_IP_HEADER                                                //
+//                                                                //
+// Written by Francois Fleuret                                    //
+// Contact <francois.fleuret@idiap.ch> for comments & bug reports //
+//                                                                //
+// END_IP_HEADER                                                  //
+////////////////////////////////////////////////////////////////////
+
+#include "vision_problem_10.h"
+#include "shape.h"
+
+VisionProblem_10::VisionProblem_10() { }
+
+void VisionProblem_10::generate(int label, Vignette *vignette) {
+  int nb_shapes = 4;
+  int xs[nb_shapes], ys[nb_shapes];
+
+  int error;
+  do {
+    do {
+      if(label == 1) {
+        scalar_t alpha = drand48() * 2 * M_PI;
+        scalar_t radius = drand48() * Vignette::width / 2;
+        scalar_t xc = drand48() * Vignette::width;
+        scalar_t yc = drand48() * Vignette::height;
+        for(int n = 0; n < nb_shapes; n++) {
+          xs[n] = int(xc + part_size/2 +
+                      radius * sin(alpha + n * 2 * M_PI / scalar_t(nb_shapes)));
+          ys[n] = int(yc + part_size/2 +
+                      radius * cos(alpha + n * 2 * M_PI / scalar_t(nb_shapes)));
+        }
+      } else {
+        for(int n = 0; n < nb_shapes; n++) {
+          xs[n] = int(drand48() * Vignette::width);
+          ys[n] = int(drand48() * Vignette::height);
+        }
+      }
+    } while(cluttered_shapes(part_size, nb_shapes, xs, ys));
+
+    vignette->clear();
+
+    Shape shape;
+    error = 0;
+    for(int n = 0; n < nb_shapes; n++) {
+      if(n == 0) {
+        shape.randomize(part_size / 2, part_hole_size / 2);
+      }
+      error |= shape.overwrites(vignette, xs[n], ys[n]);
+      shape.draw(vignette, xs[n], ys[n]);
+    }
+  } while(error);
+}
diff --git a/vision_problem_10.h b/vision_problem_10.h
new file mode 100644
index 0000000..e9b3baf
--- /dev/null
+++ b/vision_problem_10.h
@@ -0,0 +1,39 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef VISION_PROBLEM_10_H
+#define VISION_PROBLEM_10_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_10 : public VignetteGenerator {
+  static const int part_size = Vignette::width / 6;
+  static const int part_hole_size = Vignette::width / 64;
+public:
+  VisionProblem_10();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_14.cc b/vision_problem_14.cc
new file mode 100644
index 0000000..cbabafe
--- /dev/null
+++ b/vision_problem_14.cc
@@ -0,0 +1,91 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_14.h"
+#include "shape.h"
+
+VisionProblem_14::VisionProblem_14() { }
+
+void VisionProblem_14::sample_shapes_positions_aligned(int nb_shapes, int *xs, int *ys) {
+  for(int n = 0; n < nb_shapes - 1; n++) {
+    xs[n] = int(drand48() * (Vignette::width - part_size + 1)) + part_size/2;
+    ys[n] = int(drand48() * (Vignette::height - part_size + 1)) + part_size/2;
+  }
+  scalar_t alpha = drand48();
+  xs[nb_shapes - 1] = int(alpha * xs[0] + (1 - alpha) * xs[1]);
+  ys[nb_shapes - 1] = int(alpha * ys[0] + (1 - alpha) * ys[1]);
+}
+
+void VisionProblem_14::sample_shapes_positions_uniformly(int nb_shapes, int *xs, int *ys) {
+  for(int n = 0; n < nb_shapes; n++) {
+    xs[n] = int(drand48() * (Vignette::width - part_size + 1)) + part_size/2;
+    ys[n] = int(drand48() * (Vignette::height - part_size + 1)) + part_size/2;
+  }
+}
+
+int VisionProblem_14::there_is_an_alignment(scalar_t dist_threshold, int nb_shapes, int *xs, int *ys) {
+  for(int a = 0; a < nb_shapes; a++) {
+    for(int b = 0; b < nb_shapes; b++) {
+      for(int c = 0; c < b; c++) {
+        if(a != b && a != c) {
+          if(point_in_band(scalar_t(xs[a]), scalar_t(ys[a]),
+                           scalar_t(xs[b]), scalar_t(ys[b]),
+                           scalar_t(xs[c]), scalar_t(ys[c]),
+                           dist_threshold)) return 1;
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+
+void VisionProblem_14::generate(int label, Vignette *vignette) {
+  int nb_shapes = 3;
+  int xs[nb_shapes], ys[nb_shapes];
+  int error;
+
+  do {
+    if(label) {
+      sample_shapes_positions_aligned(nb_shapes, xs, ys);
+    } else {
+      do {
+        sample_shapes_positions_uniformly(nb_shapes, xs, ys);
+      } while(there_is_an_alignment(Vignette::width/20, 3, xs, ys));
+    }
+
+    vignette->clear();
+
+    Shape shape;
+
+    error = 0;
+    for(int n = 0; n < nb_shapes; n++) {
+      if(n == 0) {
+        shape.randomize(part_size/2, hole_size/2);
+      }
+      error |= shape.overwrites(vignette, xs[n], ys[n]);
+      shape.draw(vignette, xs[n], ys[n]);
+    }
+  } while(error);
+}
diff --git a/vision_problem_14.h b/vision_problem_14.h
new file mode 100644
index 0000000..c1598ff
--- /dev/null
+++ b/vision_problem_14.h
@@ -0,0 +1,42 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_14_H
+#define VISION_PROBLEM_14_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_14 : public VignetteGenerator {
+  static const int part_size = Vignette::width / 6;
+  static const int hole_size = Vignette::width / 64;
+  void sample_shapes_positions_aligned(int nb_shapes, int *xs, int *ys);
+  void sample_shapes_positions_uniformly(int nb_shapes, int *xs, int *ys);
+  int there_is_an_alignment(scalar_t dist_threshold, int nb_shapes,
+                            int *xs, int *ys);
+public:
+  VisionProblem_14();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_15.cc b/vision_problem_15.cc
new file mode 100644
index 0000000..1747cb4
--- /dev/null
+++ b/vision_problem_15.cc
@@ -0,0 +1,61 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_15.h"
+#include "shape.h"
+
+VisionProblem_15::VisionProblem_15() { }
+
+void VisionProblem_15::generate(int label, Vignette *vignette) {
+  int nb_shapes = 4;
+  int xs[nb_shapes], ys[nb_shapes];
+
+  int error;
+  do {
+    do {
+      scalar_t alpha = drand48() * 2 * M_PI;
+      scalar_t radius = drand48() * Vignette::width / 2;
+      scalar_t xc = drand48() * Vignette::width;
+      scalar_t yc = drand48() * Vignette::height;
+      for(int n = 0; n < nb_shapes; n++) {
+        xs[n] = int(xc + part_size/2 +
+                    radius * sin(alpha + n * 2 * M_PI / scalar_t(nb_shapes)));
+        ys[n] = int(yc + part_size/2 +
+                    radius * cos(alpha + n * 2 * M_PI / scalar_t(nb_shapes)));
+      }
+    } while(cluttered_shapes(part_size, nb_shapes, xs, ys));
+
+    vignette->clear();
+
+    Shape shape;
+    error = 0;
+    for(int n = 0; n < nb_shapes; n++) {
+      if(n == 0 || label == 0) {
+        shape.randomize(part_size / 2, part_hole_size / 2);
+      }
+      error |= shape.overwrites(vignette, xs[n], ys[n]);
+      shape.draw(vignette, xs[n], ys[n]);
+    }
+  } while(error);
+}
diff --git a/vision_problem_15.h b/vision_problem_15.h
new file mode 100644
index 0000000..b842dbd
--- /dev/null
+++ b/vision_problem_15.h
@@ -0,0 +1,38 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_15_H
+#define VISION_PROBLEM_15_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_15 : public VignetteGenerator {
+  static const int part_size = Vignette::width / 6;
+  static const int part_hole_size = Vignette::width / 64;
+public:
+  VisionProblem_15();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_16.cc b/vision_problem_16.cc
new file mode 100644
index 0000000..06b9046
--- /dev/null
+++ b/vision_problem_16.cc
@@ -0,0 +1,62 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_16.h"
+#include "shape.h"
+
+VisionProblem_16::VisionProblem_16() { }
+
+void VisionProblem_16::generate(int label, Vignette *vignette) {
+  int error;
+
+  int nb_shapes = 6;
+  int xs[nb_shapes], ys[nb_shapes];
+  Shape shape1, shape2;
+  shape1.randomize(part_size / 2, hole_size / 2);
+  shape2.copy(&shape1);
+
+  if(label == 1) {
+    shape2.symmetrize(0.0, 1.0);
+  }
+
+  do {
+    vignette->clear();
+    error = 0;
+
+    // First half of the shapes are random
+    for(int n = 0; n < nb_shapes/2; n++) {
+      xs[n] = int(drand48() * (Vignette::width - part_size + 1)) + part_size/2;
+      ys[n] = int(drand48() * (Vignette::height - part_size + 1)) + part_size/2;
+      error |= shape1.overwrites(vignette, xs[n], ys[n]);
+      shape1.draw(vignette, xs[n], ys[n]);
+    }
+
+    for(int n = nb_shapes/2; n < nb_shapes; n++) {
+      xs[n] = Vignette::width - xs[n - nb_shapes/2];
+      ys[n] = ys[n - nb_shapes/2];
+      error |= shape2.overwrites(vignette, xs[n], ys[n]);
+      shape2.draw(vignette, xs[n], ys[n]);
+    }
+  } while(error);
+}
diff --git a/vision_problem_16.h b/vision_problem_16.h
new file mode 100644
index 0000000..1799c31
--- /dev/null
+++ b/vision_problem_16.h
@@ -0,0 +1,38 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_16_H
+#define VISION_PROBLEM_16_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_16 : public VignetteGenerator {
+  static const int part_size = Vignette::width / 6;
+  static const int hole_size = Vignette::width / 64;
+public:
+  VisionProblem_16();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_19.cc b/vision_problem_19.cc
new file mode 100644
index 0000000..e5b6dfb
--- /dev/null
+++ b/vision_problem_19.cc
@@ -0,0 +1,73 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_19.h"
+#include "shape.h"
+
+VisionProblem_19::VisionProblem_19() { }
+
+void VisionProblem_19::generate(int label, Vignette *vignette) {
+  int nb_shapes = 2;
+  int xs[nb_shapes], ys[nb_shapes];
+  scalar_t scales[nb_shapes], angles[nb_shapes];
+  Shape shapes[nb_shapes];
+
+  int error;
+  do {
+
+    scalar_t max_scale = -1;
+
+    for(int n = 0; n < nb_shapes; n++) {
+      xs[n] = int(drand48() * Vignette::width);
+      ys[n] = int(drand48() * Vignette::height);
+
+      scales[n] = 1.0 + 3.0 * drand48();
+
+      if(n == 0 || scales[n] > max_scale) max_scale = scales[n];
+
+      angles[n] = 0;
+    }
+
+    for(int n = 0; n < nb_shapes; n++) {
+      if(n == 0 || label == 0) {
+        shapes[n].randomize(max_scale * part_size / 2, max_scale * hole_size/2);
+      } else {
+        shapes[n].copy(&shapes[0]);
+      }
+    }
+
+    for(int n = 0; n < nb_shapes; n++) {
+      shapes[n].scale(scales[n] / max_scale);
+      shapes[n].rotate(angles[n]);
+    }
+
+    vignette->clear();
+
+    error = 0;
+    for(int n = 0; n < nb_shapes; n++) {
+      error |= shapes[n].overwrites(vignette, xs[n], ys[n]);
+      shapes[n].draw(vignette, xs[n], ys[n]);
+    }
+  } while(error);
+}
diff --git a/vision_problem_19.h b/vision_problem_19.h
new file mode 100644
index 0000000..794b28e
--- /dev/null
+++ b/vision_problem_19.h
@@ -0,0 +1,38 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_19_H
+#define VISION_PROBLEM_19_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_19 : public VignetteGenerator {
+  static const int part_size = Vignette::width / 6;
+  static const int hole_size = Vignette::width / 64;
+public:
+  VisionProblem_19();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_22.cc b/vision_problem_22.cc
new file mode 100644
index 0000000..edcf595
--- /dev/null
+++ b/vision_problem_22.cc
@@ -0,0 +1,61 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_22.h"
+#include "shape.h"
+
+VisionProblem_22::VisionProblem_22() { }
+
+void VisionProblem_22::sample_shapes_positions_aligned(int nb_shapes, int *xs, int *ys) {
+  for(int n = 0; n < nb_shapes - 1; n++) {
+    xs[n] = int(drand48() * (Vignette::width - part_size + 1)) + part_size/2;
+    ys[n] = int(drand48() * (Vignette::height - part_size + 1)) + part_size/2;
+  }
+  scalar_t alpha = drand48();
+  xs[nb_shapes - 1] = int(alpha * xs[0] + (1 - alpha) * xs[1]);
+  ys[nb_shapes - 1] = int(alpha * ys[0] + (1 - alpha) * ys[1]);
+}
+
+void VisionProblem_22::generate(int label, Vignette *vignette) {
+  int nb_shapes = 3;
+  int xs[nb_shapes], ys[nb_shapes];
+  int error;
+
+  do {
+    sample_shapes_positions_aligned(nb_shapes, xs, ys);
+
+    vignette->clear();
+
+    Shape shape;
+
+    error = 0;
+    for(int n = 0; n < nb_shapes; n++) {
+      if(n == 0 || label == 0) {
+        shape.randomize(part_size/2, hole_size/2);
+      }
+      error |= shape.overwrites(vignette, xs[n], ys[n]);
+      shape.draw(vignette, xs[n], ys[n]);
+    }
+  } while(error);
+}
diff --git a/vision_problem_22.h b/vision_problem_22.h
new file mode 100644
index 0000000..f7cbdbe
--- /dev/null
+++ b/vision_problem_22.h
@@ -0,0 +1,39 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_22_H
+#define VISION_PROBLEM_22_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_22 : public VignetteGenerator {
+  static const int part_size = Vignette::width / 6;
+  static const int hole_size = Vignette::width / 64;
+  void sample_shapes_positions_aligned(int nb_shapes, int *xs, int *ys);
+public:
+  VisionProblem_22();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_23.cc b/vision_problem_23.cc
new file mode 100644
index 0000000..96f3fd2
--- /dev/null
+++ b/vision_problem_23.cc
@@ -0,0 +1,125 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_23.h"
+#include "shape.h"
+
+
+VisionProblem_23::VisionProblem_23() { }
+
+void VisionProblem_23::generate(int label, Vignette *vignette) {
+  int x_big, y_big, x_small, y_small;
+  Shape big_shape, small_shape;
+
+  int error;
+  do {
+
+    vignette->clear();
+
+    error = 0;
+
+    big_shape.randomize(big_part_size / 2, big_part_hole_size / 2);
+
+    do {
+      x_big = int(drand48() * Vignette::width);
+      y_big = int(drand48() * Vignette::height);
+    } while(big_shape.overwrites(vignette, x_big, y_big));
+
+    if(!error) {
+      big_shape.draw(vignette, x_big, y_big);
+
+      if(label) {
+        // We fill outside
+        vignette->fill(x_big, y_big, 128);
+        vignette->switch_values(128, 255);
+        // Find a location for a small shape inside
+        int nb_attempts = 0;
+        do {
+          small_shape.randomize(small_part_size / 2, small_part_hole_size / 2);
+          x_small = int(drand48() * Vignette::width);
+          y_small = int(drand48() * Vignette::height);
+          error = small_shape.overwrites(vignette, x_small, y_small);
+          nb_attempts++;
+        } while(error && nb_attempts < 10);
+
+        if(!error) {
+          // Found it, unfill outside, fill inside and draw
+          vignette->replace_value(128, 255);
+          vignette->fill(x_big, y_big, 128);
+
+          small_shape.draw(vignette, x_small, y_small);
+
+          int nb_attempts = 0;
+          do {
+            small_shape.randomize(small_part_size / 2, small_part_hole_size / 2);
+            x_small = int(drand48() * Vignette::width);
+            y_small = int(drand48() * Vignette::height);
+            error = small_shape.overwrites(vignette, x_small, y_small);
+            nb_attempts++;
+          } while(error && nb_attempts < 10);
+          if(!error) {
+            // Found it, unfill and draw
+            vignette->replace_value(128, 255);
+            small_shape.draw(vignette, x_small, y_small);
+          }
+        }
+      } else {
+        vignette->fill(x_big, y_big, 128);
+
+        if(drand48() < 0.5) {
+          vignette->switch_values(128, 255);
+        }
+
+        int nb_attempts = 0;
+        do {
+          small_shape.randomize(small_part_size / 2, small_part_hole_size / 2);
+          x_small = int(drand48() * Vignette::width);
+          y_small = int(drand48() * Vignette::height);
+          error = small_shape.overwrites(vignette, x_small, y_small);
+          nb_attempts++;
+        } while(error && nb_attempts < 10);
+
+        if(!error) {
+          small_shape.draw(vignette, x_small, y_small);
+          vignette->fill(x_small, y_small, 128);
+          int nb_attempts = 0;
+          do {
+            small_shape.randomize(small_part_size / 2, small_part_hole_size / 2);
+            x_small = int(drand48() * Vignette::width);
+            y_small = int(drand48() * Vignette::height);
+            error = small_shape.overwrites(vignette, x_small, y_small);
+            nb_attempts++;
+          } while(error && nb_attempts < 10);
+
+          if(!error) {
+            small_shape.draw(vignette, x_small, y_small);
+            vignette->replace_value(128, 255);
+          }
+        }
+      }
+
+    }
+
+  } while(error);
+}
diff --git a/vision_problem_23.h b/vision_problem_23.h
new file mode 100644
index 0000000..ec0665c
--- /dev/null
+++ b/vision_problem_23.h
@@ -0,0 +1,40 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_23_H
+#define VISION_PROBLEM_23_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_23 : public VignetteGenerator {
+  static const int small_part_size = Vignette::width/6;
+  static const int small_part_hole_size = Vignette::width/64;
+  static const int big_part_size = (Vignette::width * 3)/4;
+  static const int big_part_hole_size = Vignette::width / 3;
+public:
+  VisionProblem_23();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_4.cc b/vision_problem_4.cc
new file mode 100644
index 0000000..a49c296
--- /dev/null
+++ b/vision_problem_4.cc
@@ -0,0 +1,73 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_4.h"
+#include "shape.h"
+
+VisionProblem_4::VisionProblem_4() { }
+
+void VisionProblem_4::generate(int label, Vignette *vignette) {
+  int x_big, y_big, x_small, y_small;
+  Shape big_shape, small_shape;
+
+  int error;
+  do {
+
+    vignette->clear();
+
+    error = 0;
+
+    big_shape.randomize(big_part_size / 2, big_part_hole_size / 2);
+
+    do {
+      x_big = int(drand48() * Vignette::width);
+      y_big = int(drand48() * Vignette::height);
+    } while(big_shape.overwrites(vignette, x_big, y_big));
+
+    if(!error) {
+      big_shape.draw(vignette, x_big, y_big);
+
+      vignette->fill(x_big, y_big, 128);
+
+      if(label) {
+        vignette->switch_values(128, 255);
+      }
+
+      int nb_attempts = 0;
+      do {
+        small_shape.randomize(small_part_size / 2, small_part_hole_size / 2);
+        x_small = int(drand48() * Vignette::width);
+        y_small = int(drand48() * Vignette::height);
+        error = small_shape.overwrites(vignette, x_small, y_small);
+        nb_attempts++;
+      } while(error && nb_attempts < 10);
+
+      if(!error) {
+        vignette->replace_value(128, 255);
+        small_shape.draw(vignette, x_small, y_small);
+      }
+    }
+
+  } while(error);
+}
diff --git a/vision_problem_4.h b/vision_problem_4.h
new file mode 100644
index 0000000..a0ecf85
--- /dev/null
+++ b/vision_problem_4.h
@@ -0,0 +1,40 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_4_H
+#define VISION_PROBLEM_4_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_4 : public VignetteGenerator {
+  static const int small_part_size = Vignette::width/6;
+  static const int small_part_hole_size = Vignette::width/64;
+  static const int big_part_size = (Vignette::width * 3)/4;
+  static const int big_part_hole_size = Vignette::width / 3;
+public:
+  VisionProblem_4();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_7.cc b/vision_problem_7.cc
new file mode 100644
index 0000000..0b006bf
--- /dev/null
+++ b/vision_problem_7.cc
@@ -0,0 +1,50 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_7.h"
+#include "shape.h"
+
+VisionProblem_7::VisionProblem_7() { }
+
+void VisionProblem_7::generate(int label, Vignette *vignette) {
+  int nb_shapes;
+
+  nb_shapes = 6;
+
+  vignette->clear();
+  Shape shape;
+
+  for(int s = 0; s < nb_shapes; s++) {
+    if((label == 0 && s%2 == 0) || (label == 1 && s%3 == 0)) {
+      shape.randomize(part_size, part_hole_size);
+    }
+
+    int xs, ys;
+    do {
+      xs = int(drand48() * Vignette::width);
+      ys = int(drand48() * Vignette::height);
+    } while(shape.overwrites(vignette, xs, ys));
+    shape.draw(vignette, xs, ys);
+  }
+}
diff --git a/vision_problem_7.h b/vision_problem_7.h
new file mode 100644
index 0000000..1e7bb98
--- /dev/null
+++ b/vision_problem_7.h
@@ -0,0 +1,38 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_7_H
+#define VISION_PROBLEM_7_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_7 : public VignetteGenerator {
+  static const int part_size = Vignette::width/6;
+  static const int part_hole_size = Vignette::width/64;
+public:
+  VisionProblem_7();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_problem_9.cc b/vision_problem_9.cc
new file mode 100644
index 0000000..2be5c8d
--- /dev/null
+++ b/vision_problem_9.cc
@@ -0,0 +1,74 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "vision_problem_9.h"
+#include "shape.h"
+
+VisionProblem_9::VisionProblem_9() { }
+
+void VisionProblem_9::generate(int label, Vignette *vignette) {
+  int nb_shapes = 3;
+  int xs[nb_shapes], ys[nb_shapes];
+  Shape big_shape, small_shape;
+
+  int error;
+
+  do {
+    scalar_t x1 = int(drand48() * Vignette::width);
+    scalar_t y1 = int(drand48() * Vignette::height);
+    scalar_t x2 = int(drand48() * Vignette::width);
+    scalar_t y2 = int(drand48() * Vignette::height);
+    scalar_t alpha = 0.25 + 0.5 * drand48();
+
+    big_shape.randomize(part_size, hole_size);
+    small_shape.copy(&big_shape);
+    small_shape.scale(0.5);
+
+    if(label == 0) {
+      xs[0] = int(x1); ys[0] = int(y1);
+      xs[1] = int(x2); ys[1] = int(y2);
+      xs[2] = int(alpha * x1 + (1 - alpha) * x2); ys[2] = int(alpha * y1 + (1 - alpha) * y2);
+    } else {
+      xs[0] = int(x1); ys[0] = int(y1);
+      xs[1] = int(alpha * x1 + (1 - alpha) * x2); ys[1] = int(alpha * y1 + (1 - alpha) * y2);
+      xs[2] = int(x2); ys[2] = int(y2);
+    }
+
+    vignette->clear();
+
+    error = 0;
+    for(int n = 0; n < nb_shapes; n++) {
+      if(n < 2) {
+        error |= small_shape.overwrites(vignette, xs[n], ys[n]);
+        small_shape.draw(vignette, xs[n], ys[n]);
+      } else {
+        error |= big_shape.overwrites(vignette, xs[n], ys[n]);
+        big_shape.draw(vignette, xs[n], ys[n]);
+      }
+      vignette->fill(xs[n], ys[n], 128);
+    }
+
+    vignette->replace_value(128, 255);
+  } while(error);
+}
diff --git a/vision_problem_9.h b/vision_problem_9.h
new file mode 100644
index 0000000..b785da1
--- /dev/null
+++ b/vision_problem_9.h
@@ -0,0 +1,38 @@
+/*
+ *  svrt is the ``Synthetic Visual Reasoning Test'', an image
+ *  generator for evaluating classification performance of machine
+ *  learning systems, humans and primates.
+ *
+ *  Copyright (c) 2009 Idiap Research Institute, http://www.idiap.ch/
+ *  Written by Francois Fleuret <francois.fleuret@idiap.ch>
+ *
+ *  This file is part of svrt.
+ *
+ *  svrt is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 3 as
+ *  published by the Free Software Foundation.
+ *
+ *  svrt is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with selector.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#ifndef VISION_PROBLEM_9_H
+#define VISION_PROBLEM_9_H
+
+#include "vignette_generator.h"
+#include "vision_problem_tools.h"
+
+class VisionProblem_9 : public VignetteGenerator {
+  static const int part_size = Vignette::width / 6;
+  static const int hole_size = Vignette::width / 64;
+public:
+  VisionProblem_9();
+  virtual void generate(int label, Vignette *vignette);
+};
+
+#endif
diff --git a/vision_test.cc b/vision_test.cc
index 3a445e7..28c29d3 100644
--- a/vision_test.cc
+++ b/vision_test.cc
@@ -45,16 +45,26 @@ using namespace std;
 #include "vision_problem_1.h"
 #include "vision_problem_2.h"
 #include "vision_problem_3.h"
+#include "vision_problem_4.h"
 #include "vision_problem_5.h"
 #include "vision_problem_6.h"
+#include "vision_problem_7.h"
 #include "vision_problem_8.h"
+#include "vision_problem_9.h"
+#include "vision_problem_10.h"
 #include "vision_problem_11.h"
 #include "vision_problem_12.h"
 #include "vision_problem_13.h"
+#include "vision_problem_14.h"
+#include "vision_problem_15.h"
+#include "vision_problem_16.h"
 #include "vision_problem_17.h"
 #include "vision_problem_18.h"
+#include "vision_problem_19.h"
 #include "vision_problem_20.h"
 #include "vision_problem_21.h"
+#include "vision_problem_22.h"
+#include "vision_problem_23.h"
 
 //////////////////////////////////////////////////////////////////////
 
@@ -102,27 +112,27 @@ int main(int argc, char **argv) {
   case 3:
     generator = new VisionProblem_3();
     break;
-  // case 4:
-    // generator = new VisionProblem_4();
-    // break;
+  case 4:
+    generator = new VisionProblem_4();
+    break;
   case 5:
     generator = new VisionProblem_5();
     break;
   case 6:
     generator = new VisionProblem_6();
     break;
-  // case 7:
-    // generator = new VisionProblem_7();
-    // break;
+  case 7:
+    generator = new VisionProblem_7();
+    break;
   case 8:
     generator = new VisionProblem_8();
     break;
-  // case 9:
-    // generator = new VisionProblem_9();
-    // break;
-  // case 10:
-    // generator = new VisionProblem_10();
-    // break;
+  case 9:
+    generator = new VisionProblem_9();
+    break;
+  case 10:
+    generator = new VisionProblem_10();
+    break;
   case 11:
     generator = new VisionProblem_11();
     break;
@@ -132,36 +142,36 @@ int main(int argc, char **argv) {
   case 13:
     generator = new VisionProblem_13();
     break;
-  // case 14:
-    // generator = new VisionProblem_14();
-    // break;
-  // case 15:
-    // generator = new VisionProblem_15();
-    // break;
-  // case 16:
-    // generator = new VisionProblem_16();
-    // break;
+  case 14:
+    generator = new VisionProblem_14();
+    break;
+  case 15:
+    generator = new VisionProblem_15();
+    break;
+  case 16:
+    generator = new VisionProblem_16();
+    break;
   case 17:
     generator = new VisionProblem_17();
     break;
   case 18:
     generator = new VisionProblem_18();
     break;
-  // case 19:
-    // generator = new VisionProblem_19();
-    // break;
+  case 19:
+    generator = new VisionProblem_19();
+    break;
   case 20:
     generator = new VisionProblem_20();
     break;
   case 21:
     generator = new VisionProblem_21();
     break;
-  // case 22:
-    // generator = new VisionProblem_22();
-    // break;
-  // case 23:
-    // generator = new VisionProblem_23();
-    // break;
+  case 22:
+    generator = new VisionProblem_22();
+    break;
+  case 23:
+    generator = new VisionProblem_23();
+    break;
   default:
     cerr << "Can not find problem "
          << global.problem_number