1 % -*- mode: latex; mode: reftex; mode: auto-fill; mode: flyspell; coding: utf-8; tex-command: "pdflatex.sh"; TeX-master: "lbdl.tex" -*-
3 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
12 \usepackage[us]{datetime}
13 \newdateformat{dotdate}{\THEYEAR.\twodigit{\THEMONTH}.\twodigit{\THEDAY}}
16 \makeindex[intoc,columns=1]
19 \setlist[itemize]{leftmargin=0pt,itemindent=1em,itemsep=2ex}
20 \setlist{nosep} % or \setlist{noitemsep} to leave space around whole list
22 \usepackage[utf8]{inputenc}
23 \usepackage[T1]{fontenc}
24 \usepackage[osf]{libertine}
25 \usepackage{microtype}
31 bookmarksnumbered=false,
40 ]{hyperref} % PDF meta-information specification
44 \usepackage[object=vectorian]{pgfornament}
48 \centerline{\pgfornament[anchor=center,ydelta=0pt,width=2cm]{82}}
53 \AddToHook{cmd/section/before}{\clearpage}
54 \usepackage[section]{placeins}
57 \def\wordfig{Figure\xspace}
58 \def\wordtable{Table\xspace}
59 \def\wordchap{Chapter\xspace}
62 \let\oldendcenter\endcenter
63 \renewenvironment{center}{\setlength\topsep{0pt}\oldcenter}{\oldendcenter}
65 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
67 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
69 \setlength{\thinmuskip}{1mu} % by default it is equal to 3 mu
70 \setlength{\medmuskip}{1mu} % by default it is equal to 4 mu
71 \setlength{\thickmuskip}{3mu} % by default it is equal to 5 mu
74 \DeclareFontEncoding{LS1}{}{}
75 \DeclareFontSubstitution{LS1}{stix}{m}{n}
76 \DeclareMathAlphabet{\mathcal}{LS1}{stixscr}{m}{n}
79 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
81 \def\given{\,\middle\vert\,}
82 \newcommand{\proba}{{P}}
83 \newcommand{\seq}{{S}}
84 \newcommand{\expect}{\mathds{E}}
85 \newcommand{\variance}{\mathds{V}}
86 \newcommand{\empexpect}{\hat{\mathds{E}}}
87 \newcommand{\mutinf}{\mathds{I}}
88 \newcommand{\empmutinf}{\hat{\mathds{I}}}
89 \newcommand{\entropy}{\mathds{H}}
90 \newcommand{\empentropy}{\hat{\mathds{H}}}
91 \newcommand{\ganG}{\mathbf{G}}
92 \newcommand{\ganD}{\mathbf{D}}
93 \newcommand{\ganF}{\mathbf{F}}
95 \newcommand{\dkl}{\mathds{D}_{\mathsf{KL}}}
96 \newcommand{\djs}{\mathds{D}_{\mathsf{JS}}}
98 \newcommand*{\vertbar}{\rule[-1ex]{0.5pt}{2.5ex}}
99 \newcommand*{\horzbar}{\rule[.5ex]{2.5ex}{0.5pt}}
101 \def\positionalencoding{\operatorname{pos-enc}}
102 \def\concat{\operatorname{concat}}
103 \def\crossentropy{\LL_{\operatorname{ce}}}
105 \def\embedding{\operatorname{embed}}
106 \def\mha{\operatorname{mha}}
107 \def\layernorm{\operatorname{layernorm}}
108 \def\batchnorm{\operatorname{batchnorm}}
109 \def\fullyconnected{\operatorname{fully-conn}}
110 \def\softargmax{\operatorname{softargmax}}
111 \def\selfattention{\operatorname{self-att}}
112 \def\crossattention{\operatorname{cross-att}}
113 \def\attention{\operatorname{att}}
114 \def\relu{\operatorname{relu}}
115 \def\gelu{\operatorname{gelu}}
116 \def\dropout{\operatorname{dropout}}
117 \def\resblock{\operatorname{resblock}}
118 \def\dresblock{\operatorname{dresblock}}
119 \def\reshape{\operatorname{reshape}}
120 \def\convtwod{\operatorname{conv-2d}}
121 \def\maxpool{\operatorname{maxpool}}
122 \def\avgpool{\operatorname{avgpool}}
123 %\def\samax{\Upsilon}
124 %\def\samax{\operatorname{samax}}
125 \def\sigmoid{\operatorname{sigm}}
126 \def\sample{\operatorname{sample}}
127 \def\diag{\operatorname{diag}}
128 \def\sign{\operatorname{sign}}
129 \def\argmax{\operatornamewithlimits{argmax}}
130 \def\argmin{\operatornamewithlimits{argmin}}
132 %\usepackage{oldgerm}
135 %\newcommand{\li}[1]{^{\textgoth{#1}}}
136 \newcommand{\li}[1]{^{\scalebox{.5}{\textbf{#1}}}}
137 %% \newcommand{\li}[1]{^{\textbf{#1}}}
138 %\newcommand{\li}[1]{{|#1}}
139 \newcommand{\DATAVAR}{\mathbf{{\cal D}}}
140 \newcommand{\DATAVAL}{\mathbf{d}}
141 \newcommand{\BD}{\mathbf{D}}
142 \newcommand{\LL}{\mathcal{L}}
143 \newcommand{\Ll}{\mathcal{l}}
144 \newcommand{\RR}{\mathbb{R}}
145 \newcommand{\Lh}{\mathcal{h}}
146 \newcommand{\transpose}{^{\top}}
148 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
150 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
153 \usetikzlibrary{arrows,arrows.meta,calc}
154 \usetikzlibrary{patterns,backgrounds}
155 \usetikzlibrary{positioning,fit}
156 \usetikzlibrary{shapes.geometric,shapes.multipart}
157 \usetikzlibrary{patterns.meta,decorations.pathreplacing,calligraphy}
158 \usetikzlibrary{tikzmark}
159 \usetikzlibrary{decorations.pathmorphing}
161 % remove the "There is no ... in font nullfont!" errors
162 \AtBeginEnvironment{tikzpicture}{\tracinglostchars=0\relax}
167 \definecolor{paramcolor}{rgb}{0.85,0.85,0.85}
177 \newlength{\layergap}
178 \setlength{\layergap}{2pt}
179 \newlength{\layerthickness}
180 \setlength{\layerthickness}{12pt}
181 \newlength{\layerwidth}
182 \setlength{\layerwidth}{4.5em}
184 \newlength{\diminfoshift}
185 \setlength{\diminfoshift}{70pt}
188 >={Straight Barb[angle'=80,scale=1.2]},
190 %% background rectangle/.style={fill=paper},
191 %% show background rectangle,
192 %every text node part/.style={align=center},
193 rounded corners=0.5pt,
194 curly brace/.style={sharp corners,very thick,decoration={calligraphic brace,amplitude=0.20cm},decorate},
197 %% on layer=background,
199 draw=white,line width=2pt,-,%shorten <=1pt,shorten >=1pt,
202 operator/.style={draw=black,inner sep=1pt},
203 next/.style={above=##1\layergap of \tikzlastnode},
205 prev/.style={below=##1\layergap of \tikzlastnode},
207 var/.style={inner sep=2pt},
208 layer/.style={operator,minimum width=\layerwidth,minimum height=\layerthickness,text depth=1pt,text height=1.3ex},
209 layer large/.style={layer,minimum height=1.75\layerthickness},
210 layer small/.style={layer,minimum width=\layerthickness},
212 draw=black,line width=0.4pt,dash pattern=on 0.4pt off 2pt,
213 %% draw=black!50,line width=0.2pt,-,
214 shorten >=2pt,shorten <=2pt,
216 block definition/.style={draw=black,inner sep=2\layergap,dash pattern=on 2.5pt off 0.5pt},
219 inner sep=\layergap, dash pattern=on 2.5pt off 0.35pt,
223 ]south east:$\times ##1$},
229 \draw[black,very thin]
230 ($(path picture bounding box.south west)+(1pt,6pt)$)--($(path picture bounding box.south east)+(-1pt,6pt)$)
232 node[midway,yshift=-15.5pt] {\scalebox{.5}{##1}};
238 %% preaction={fill=white},
239 %% pattern color=black!15,
240 %% pattern={Lines[line width=0.5pt,angle=-45,distance=1pt]}
242 meta param/.style={label={[%
247 ]south east:{\tiny\color{blue}##1}}},
251 \newcommand{\diminfo}[3]{%
252 \coordinate (t) at ($(#2.north)+(\diminfoshift,0.5\layergap)$);
253 \node[inner sep=0pt,yshift=-0.5pt] (s) at (#1.north east-|t) {\tiny #3};
254 \draw[info line] (#1.north east|-s)--(s);
257 \newcommand{\defop}[2]{%
258 %% \coordinate (BL) at ($(#1.north)+(-0.49\textwidth, 4\layergap)$);
259 %% \coordinate (BR) at ($(#1.north)+( 0.49\textwidth, 4\layergap)$);
260 %% \coordinate (TL) at ($(#2.south-|#1)+(-0.49\textwidth,-4\layergap)$);
261 %% \coordinate (TR) at ($(#2.south-|#1)+( 0.49\textwidth,-4\layergap)$);
262 \begin{pgfinterruptboundingbox}
263 \node[anchor=south west,inner sep=2pt] (label) at #1 {#2};
264 \draw[decorate,decoration={coil,amplitude=0.5pt,segment length=2pt,aspect=0}] (label.south west) -- (label.south east);
265 \end{pgfinterruptboundingbox}
268 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
273 \pgfonlayer{#1}\begingroup
274 \aftergroup\endpgfonlayer
279 %% fix for bb computation of double wires.
280 %% from https://tex.stackexchange.com/questions/130456/tikz-double-lines-are-shifted
282 only coordinates are relevant/.is choice,
283 only coordinates are relevant/.default=true,
284 only coordinates are relevant/true/.code={%
285 \tikz@addmode{\pgf@relevantforpicturesizefalse}},
286 only coordinates are relevant/false/.code={%
287 \tikz@addmode{\pgf@relevantforpicturesizetrue}}
291 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
294 % extract interval `start:end` values
295 \def\get@interval@start#1:#2\@nil{#1}
296 \def\get@interval@end#1:#2\@nil{#2}
298 \def\domainmin{\expandafter\get@interval@start\tikz@plot@domain\@nil}
299 \def\domainmax{\expandafter\get@interval@end\tikz@plot@domain\@nil}
301 \def\rangemin{\expandafter\get@interval@start\tikz@plot@range\@nil}
302 \def\rangemax{\expandafter\get@interval@end\tikz@plot@range\@nil}
305 \usepackage{pgfplots}
306 \usepgfplotslibrary{patchplots,colormaps}
307 \pgfplotsset{compat = newest}
309 \newcommand{\mygrid}[5]{%
310 \pgfmathsetmacro{\xmin}{#1+1}
311 \pgfmathsetmacro{\xmax}{#1+#3-1}
312 \pgfmathsetmacro{\ymin}{#2+1}
313 \pgfmathsetmacro{\ymax}{#2+#4-1}
314 \ifthenelse{\equal{#5}{}}
315 {\draw (#1,#2) rectangle ++(#3,#4);}
316 {\draw[fill=#5] (#1,#2) rectangle ++(#3,#4);}
317 \foreach \x in {\xmin,...,\xmax}{
318 \draw (\x,#2)-- ++(0,#4);
320 \foreach \y in {\ymin,...,\ymax}{
321 \draw (#1,\y)-- ++(#3,0);
325 \newcommand{\amatrix}[7]{%
326 \begin{tikzpicture}[scale=0.2]
327 \ifthenelse{\equal{#7}{}}
329 {\draw[draw=none,fill=#7] (#3,#4) rectangle ++(#5,#6);}
330 \mygrid{0}{0}{#1}{#2}{}
334 \newcommand{\gridcube}[3]{% 7,4,6
336 \foreach \b in { 0,...,#2 }{
337 \draw (0,\b,0)--++(#1,0,0)--++(0,0,#3);
340 \foreach \d in { 0,...,#1 }{
341 \draw (\d,0,0)--++(0,#2,0)--++(0,0,#3);
344 \foreach \hw in { 0,...,#3 }{
345 \draw (0,0,\hw)++(#1,0,0)--++(0,#2,0)--++(-#1,0,0);
349 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
351 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
353 \usepackage[square]{natbib}
354 \bibliographystyle{plainnatmodified}
356 \newcommand{\biburl}[1]{\href{#1}{pdf}}
358 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
360 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
362 \cftsetindents{part}{0em}{1.8em}
363 \cftsetindents{chapter}{0em}{1.8em}
364 \cftsetindents{section}{1.8em}{2.2em}
366 \setlength{\parindent}{0cm}
367 \setlength{\parskip}{2ex}
369 \setstocksize{15cm}{8cm}
370 \settrimmedsize{\stockheight}{\stockwidth}{*}
371 \setlrmarginsandblock{8pt}{8pt}{*}
372 \setulmarginsandblock{14pt}{26pt}{*}
373 \setheadfoot{14pt}{14pt}
374 \setheaderspaces{*}{*}{*}
376 \makepagestyle{littlebook}
377 \makeoddhead{littlebook}{}{}{}
378 \makeevenhead{littlebook}{}{}{}
379 \newcommand{\myfooter}{\footnotesize {\thepage \hskip 0.8em \raisebox{-2pt}{\vline height 8pt} \hskip 0.4em \thelastpage}}
380 \makeoddfoot{littlebook}{}{\myfooter}{}
381 \makeoddfoot{plain}{}{\myfooter}{}
382 \pagestyle{littlebook}
384 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
386 \renewcommand{\partnamefont}{\centering\sffamily\scshape\Huge}
387 \renewcommand{\partnumfont}{\sffamily\Huge}
388 \renewcommand{\parttitlefont}{\centering\sffamily\scshape\Huge}
389 \renewcommand{\beforepartskip}{\vspace*{\stretch{3}}}
390 \renewcommand{\afterpartskip}{%
391 \vspace*{\stretch{4}}
395 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
397 \makechapterstyle{Tufte}{
398 \renewcommand{\chapterheadstart}{\null \vskip3.0\onelineskip}
399 \renewcommand{\printchaptername}{\large\itshape\chaptername}
400 \renewcommand{\printchapternum}{\LARGE\thechapter \\}
401 \renewcommand{\afterchapternum}{}
402 \renewcommand{\printchaptertitle}[1]{
405 \renewcommand{\afterchaptertitle}{
406 \vskip3.5\onelineskip
410 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
412 \setsecheadstyle{\sethangfrom{\noindent ##1}\raggedright\sffamily\itshape\Large}
413 \setbeforesecskip{-.9\onelineskip}
414 \setaftersecskip{.5\onelineskip}
416 \setsubsecheadstyle{\sethangfrom{\noindent ##1}\raggedright\sffamily\large\itshape}
417 \setbeforesubsecskip{-.6\onelineskip}
418 \setaftersubsecskip{.5\onelineskip}
420 \setsubsubsecheadstyle{\sethangfrom{\noindent ##1}\raggedright\sffamily\itshape}
421 \setbeforesubsubsecskip{-.5\onelineskip}
422 \setaftersubsubsecskip{.1\onelineskip}
424 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
426 \captiontitlefont{\itshape\small}
427 \captionnamefont{\small}
428 \newcommand{\likecaption}{\color{black}\itshape\small}
432 \checkandfixthelayout
434 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
436 \newcounter{nbdrafts}
437 \setcounter{nbdrafts}{0}
439 \newcommand{\checknbdrafts}{
440 \ifnum \thenbdrafts > 0
441 \@latex@warning@no@line{*WARNING* The document contains \thenbdrafts \space draft note(s)}
443 \newcommand{\todo}[1]{\addtocounter{nbdrafts}{1}{\color{red} #1}}
445 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
446 \definecolor{paper}{rgb}{0.95,0.95,0.95}
447 \definecolor{math}{rgb}{0.0,0.5,0.0}
448 %\definecolor{links}{rgb}{0.0,0.2,0.5}
449 \definecolor{links}{rgb}{0.0,0.2,0.85}
450 %\definecolor{hlcolor}{rgb}{0.8,1.0,0.85}
452 \definecolor{blue}{rgb}{0.3,0.5,0.85}
453 \definecolor{red}{rgb}{0.65,0.0,0.0}
454 \definecolor{green}{rgb}{0.0,0.50,0.0}
455 \definecolor{dimmed}{rgb}{0.8,0.8,0.8}
456 \definecolor{orange}{rgb}{1.0,0.75,0.0}
458 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
459 % Pretty underline, taken from
460 % https://tex.stackexchange.com/questions/36894/underline-omitting-the-descenders
468 \cs_new:Npn \white_text:n #1
470 \fp_set:Nn \l_tmpa_fp {#1 * .01}
471 \llap{\textcolor{white}{\the\SOUL@syllable}\hspace{\fp_to_decimal:N \l_tmpa_fp em}}
472 \llap{\textcolor{white}{\the\SOUL@syllable}\hspace{-\fp_to_decimal:N \l_tmpa_fp em}}
474 \NewDocumentCommand{\whiten}{ m }
476 \int_step_function:nnnN {1}{1}{#1} \white_text:n
480 \NewDocumentCommand{ \prettyul }{ D<>{5} O{0.2ex} O{0.1ex} +m } {%
483 \def\SOUL@uleverysyllable{%
484 \setbox0=\hbox{\the\SOUL@syllable}%
486 \SOUL@ulunderline{\phantom{\the\SOUL@syllable}}%
490 \SOUL@setkern\SOUL@charkern
495 \SOUL@setkern\SOUL@charkern
505 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
508 \usepackage{xcolor, soul}
510 \definecolor{hlcolor}{rgb}{1.0,1.0,0.5}
511 \definecolor{ulcolor}{rgb}{0.80,0.80,0.80}
514 \newcommand{\keyterm}[2][]{%
515 \BeginAccSupp{method=escape,ActualText={#2}}%
516 \ifthenelse{\equal{#1}{}}
517 {\prettyul[2pt]{#2}\index{#2}}
518 {\prettyul[2pt]{#2}\index{#1}}%
522 \newcommand{\gizmo}{%
523 \begin{tikzpicture}[scale=0.25]
524 \draw[draw=none,fill=red] (0,0) rectangle ++(1,1);
525 \draw[draw=none,fill=blue] (1,0) rectangle ++(1,1);
526 \draw[draw=none,fill=blue] (0,1) rectangle ++(1,1);
527 \draw[draw=none,fill=red] (1,1) rectangle ++(1,1);
531 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%