1 % -*- mode: latex; mode: reftex; mode: auto-fill; mode: flyspell; coding: utf-8; -*-
3 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
12 \usepackage[us]{datetime}
13 \newdateformat{dotdate}{\THEYEAR.\twodigit{\THEMONTH}.\twodigit{\THEDAY}}
19 \setlist[itemize]{leftmargin=0pt,itemindent=1em,itemsep=2ex}
20 \setlist{nosep} % or \setlist{noitemsep} to leave space around whole list
22 \usepackage[utf8]{inputenc}
23 \usepackage[T1]{fontenc}
24 \usepackage[osf]{libertine}
25 \usepackage{microtype}
31 bookmarksnumbered=false,
40 hypertexnames=false, % to avoid errors with autonum
41 ]{hyperref} % PDF meta-information specification
45 \usepackage[object=vectorian]{pgfornament}
49 \centerline{\pgfornament[anchor=center,ydelta=0pt,width=2cm]{82}}
54 \AddToHook{cmd/section/before}{\clearpage}
55 \usepackage[section]{placeins}
58 \def\wordfig{Figure\xspace}
59 \def\wordeq{Equation\xspace}
60 \def\wordtable{Table\xspace}
61 \def\wordchap{Chapter\xspace}
64 \let\oldendcenter\endcenter
65 \renewenvironment{center}{\setlength\topsep{0pt}\oldcenter}{\oldendcenter}
68 \NewEnviron{hardcenter}{\makebox[\textwidth][c]{\BODY}}
70 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
72 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
77 \usepackage{mleftright}
79 \setlength{\thinmuskip}{1.5mu} % by default it is equal to 3 mu
80 \setlength{\medmuskip}{2mu} % by default it is equal to 4 mu
81 \setlength{\thickmuskip}{3.5mu} % by default it is equal to 5 mu
84 \DeclareFontEncoding{LS1}{}{}
85 \DeclareFontSubstitution{LS1}{stix}{m}{n}
86 \DeclareMathAlphabet{\mathcal}{LS1}{stixscr}{m}{n}
89 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
91 \newcommand{\inputgenerated}[1]{%
92 \IfFileExists{#1}{\input{#1}}{%
93 \errmessage{Cannot find "#1", compile with -shell-escape}\stop}%
96 %\newcommand{\gradient}[2]{{\nabla\!}_{#1 \mid #2}}
97 \newcommand{\gradient}[2]{{\nabla\!#1}_{\mid #2}}
99 \def\given{\,\middle\vert\,}
100 \newcommand{\proba}{{P}}
101 \newcommand{\seq}{{S}}
102 \newcommand{\expect}{\mathds{E}}
103 \newcommand{\variance}{\mathds{V}}
104 \newcommand{\empexpect}{\hat{\mathds{E}}}
105 \newcommand{\mutinf}{\mathds{I}}
106 \newcommand{\empmutinf}{\hat{\mathds{I}}}
107 \newcommand{\entropy}{\mathds{H}}
108 \newcommand{\empentropy}{\hat{\mathds{H}}}
109 \newcommand{\ganG}{\mathbf{G}}
110 \newcommand{\ganD}{\mathbf{D}}
111 \newcommand{\ganF}{\mathbf{F}}
113 \newcommand{\dkl}{\mathds{D}_{\mathsf{KL}}}
114 \newcommand{\djs}{\mathds{D}_{\mathsf{JS}}}
116 \newcommand*{\vertbar}{\rule[-1ex]{0.5pt}{2.5ex}}
117 \newcommand*{\horzbar}{\rule[.5ex]{2.5ex}{0.5pt}}
119 \def\positionalencoding{\operatorname{pos-enc}}
120 \def\concat{\operatorname{concat}}
121 \def\crossentropy{\LL_{\operatorname{ce}}}
123 \def\embedding{\operatorname{embed}}
124 \def\mha{\operatorname{mha}}
125 \def\layernorm{\operatorname{layernorm}}
126 \def\batchnorm{\operatorname{batchnorm}}
127 \def\fullyconnected{\operatorname{fully-conn}}
128 \def\softargmax{\operatorname{softargmax}}
129 \def\selfattention{\operatorname{self-att}}
130 \def\crossattention{\operatorname{cross-att}}
131 \def\attention{\operatorname{att}}
132 \def\relu{\operatorname{relu}}
133 \def\gelu{\operatorname{gelu}}
134 \def\dropout{\operatorname{dropout}}
135 \def\resblock{\operatorname{resblock}}
136 \def\dresblock{\operatorname{dresblock}}
137 \def\reshape{\operatorname{reshape}}
138 \def\convtwod{\operatorname{conv-2d}}
139 \def\maxpool{\operatorname{maxpool}}
140 \def\avgpool{\operatorname{avgpool}}
141 %\def\samax{\Upsilon}
142 %\def\samax{\operatorname{samax}}
143 \def\sigmoid{\operatorname{sigm}}
144 \def\sample{\operatorname{sample}}
145 \def\diag{\operatorname{diag}}
146 \def\sign{\operatorname{sign}}
147 \def\argmax{\operatornamewithlimits{argmax}}
148 \def\argmin{\operatornamewithlimits{argmin}}
150 %\usepackage{oldgerm}
154 \newcommand{\adaptedscale}[1]{#1}
156 %\newcommand{\li}[1]{^{\textgoth{#1}}}
157 \newcommand{\li}[1]{^{\scalebox{.5}{\textbf{#1}}}}
158 %% \newcommand{\li}[1]{^{\textbf{#1}}}
159 %\newcommand{\li}[1]{{|#1}}
160 \newcommand{\DATAVAR}{\mathbf{{\cal D}}}
161 \newcommand{\DATAVAL}{\mathbf{d}}
162 \newcommand{\BD}{\mathbf{D}}
163 \newcommand{\LL}{\mathcal{L}}
164 \newcommand{\Ll}{\mathcal{l}}
165 \newcommand{\RR}{\mathbb{R}}
166 \newcommand{\Lh}{\mathcal{h}}
167 \newcommand{\transpose}{^{\top}}
169 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
171 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
174 \usetikzlibrary{arrows,arrows.meta,calc}
175 \usetikzlibrary{patterns,backgrounds}
176 \usetikzlibrary{positioning,fit}
177 \usetikzlibrary{shapes.geometric,shapes.multipart}
178 \usetikzlibrary{patterns.meta,decorations.pathreplacing,calligraphy}
179 \usetikzlibrary{tikzmark}
180 \usetikzlibrary{decorations.pathmorphing}
182 % remove the "There is no ... in font nullfont!" errors
183 \AtBeginEnvironment{tikzpicture}{\tracinglostchars=0\relax}
188 \definecolor{operatorcolor}{rgb}{0.95,0.95,1.00}
189 \definecolor{paramcolor}{rgb}{0.8,0.8,1.0}
199 \newlength{\layergap}
200 \setlength{\layergap}{2pt}
201 \newlength{\layerthickness}
202 \setlength{\layerthickness}{12pt}
203 \newlength{\layerwidth}
204 \setlength{\layerwidth}{4.5em}
206 \newlength{\diminfoshift}
207 \setlength{\diminfoshift}{70pt}
210 >={Straight Barb[angle'=80,scale=\adaptedscale{1.2}]},
212 %% background rectangle/.style={fill=paper},
213 %% show background rectangle,
214 %every text node part/.style={align=center},
215 %rounded corners=0.5pt,
216 curly brace/.style={sharp corners,very thick,decoration={calligraphic brace,amplitude=0.20cm},decorate},
219 %% on layer=background,
221 draw=white,line width=2pt,-,%shorten <=1pt,shorten >=1pt,
224 operator/.style={draw=black!30,fill=operatorcolor,inner sep=1pt},
225 next/.style={above=##1\layergap of \tikzlastnode},
227 prev/.style={below=##1\layergap of \tikzlastnode},
229 var/.style={inner sep=2pt},
231 layer/.style={operator,minimum width=\layerwidth,minimum height=\layerthickness,text depth=1pt,text height=1.3ex},
232 layer small/.style={layer,minimum width=\layerthickness},
233 layer large/.style={layer,minimum height=1.5\layerthickness},
234 layer very large/.style={layer,minimum height=1.75\layerthickness},
236 draw=black,line width=0.4pt,dash pattern=on 0.4pt off 2pt,
237 %% draw=black!50,line width=0.2pt,-,
238 shorten >=2pt,shorten <=2pt,
240 block definition/.style={draw=black,inner sep=2\layergap,dash pattern=on 2.5pt off 0.5pt},
243 inner sep=\layergap, dash pattern=on 2.5pt off 0.8pt,
247 ]south east:$\times ##1$},
254 ($(path picture bounding box.south west)+(1pt,6pt)$)--($(path picture bounding box.south east)+(-1pt,6pt)$)
256 node[midway,yshift=-15.5pt] {\scalebox{.5}{##1}};
263 %% preaction={fill=white},
264 %% pattern color=black!15,
265 %% pattern={Lines[line width=0.5pt,angle=-45,distance=1pt]}
267 meta param/.style={label={[%
272 ]south east:{\tiny\color{blue}##1}}},
276 \newcommand{\diminfo}[3]{%
277 \coordinate (t) at ($(#2.north)+(\diminfoshift,0.5\layergap)$);
278 \node[inner sep=0pt,yshift=-0.5pt] (s) at (#1.north east-|t) {\tiny #3};
279 \draw[info line] (#1.north east|-s)--(s);
282 \newcommand{\defop}[2]{%
283 %% \coordinate (BL) at ($(#1.north)+(-0.49\textwidth, 4\layergap)$);
284 %% \coordinate (BR) at ($(#1.north)+( 0.49\textwidth, 4\layergap)$);
285 %% \coordinate (TL) at ($(#2.south-|#1)+(-0.49\textwidth,-4\layergap)$);
286 %% \coordinate (TR) at ($(#2.south-|#1)+( 0.49\textwidth,-4\layergap)$);
287 \begin{pgfinterruptboundingbox}
288 \node[anchor=south west,inner sep=2pt] (label) at #1 {#2};
289 \draw[decorate,decoration={coil,amplitude=0.5pt,segment length=2pt,aspect=0}] (label.south west) -- (label.south east);
290 \end{pgfinterruptboundingbox}
293 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
298 \pgfonlayer{#1}\begingroup
299 \aftergroup\endpgfonlayer
304 %% fix for bb computation of double wires.
305 %% from https://tex.stackexchange.com/questions/130456/tikz-double-lines-are-shifted
307 only coordinates are relevant/.is choice,
308 only coordinates are relevant/.default=true,
309 only coordinates are relevant/true/.code={%
310 \tikz@addmode{\pgf@relevantforpicturesizefalse}},
311 only coordinates are relevant/false/.code={%
312 \tikz@addmode{\pgf@relevantforpicturesizetrue}}
316 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
319 % extract interval `start:end` values
320 \def\get@interval@start#1:#2\@nil{#1}
321 \def\get@interval@end#1:#2\@nil{#2}
323 \def\domainmin{\expandafter\get@interval@start\tikz@plot@domain\@nil}
324 \def\domainmax{\expandafter\get@interval@end\tikz@plot@domain\@nil}
326 \def\rangemin{\expandafter\get@interval@start\tikz@plot@range\@nil}
327 \def\rangemax{\expandafter\get@interval@end\tikz@plot@range\@nil}
330 \usepackage{pgfplots}
331 \usepgfplotslibrary{patchplots,colormaps}
332 \pgfplotsset{compat = newest}
334 \newcommand{\mygrid}[5]{%
335 \pgfmathsetmacro{\xmin}{#1+1}
336 \pgfmathsetmacro{\xmax}{#1+#3-1}
337 \pgfmathsetmacro{\ymin}{#2+1}
338 \pgfmathsetmacro{\ymax}{#2+#4-1}
339 \ifthenelse{\equal{#5}{}}
340 {\draw (#1,#2) rectangle ++(#3,#4);}
341 {\draw[fill=#5] (#1,#2) rectangle ++(#3,#4);}
342 \foreach \x in {\xmin,...,\xmax}{
343 \draw (\x,#2)-- ++(0,#4);
345 \foreach \y in {\ymin,...,\ymax}{
346 \draw (#1,\y)-- ++(#3,0);
350 \newcommand{\amatrix}[7]{%
351 \begin{tikzpicture}[scale=\adaptedscale{0.2}]
352 \ifthenelse{\equal{#7}{}}
354 {\draw[draw=none,fill=#7] (#3,#4) rectangle ++(#5,#6);}
355 \mygrid{0}{0}{#1}{#2}{}
359 \newcommand{\gridcube}[3]{% 7,4,6
361 \foreach \b in { 0,...,#2 }{
362 \draw (0,\b,0)--++(#1,0,0)--++(0,0,#3);
365 \foreach \d in { 0,...,#1 }{
366 \draw (\d,0,0)--++(0,#2,0)--++(0,0,#3);
369 \foreach \hw in { 0,...,#3 }{
370 \draw (0,0,\hw)++(#1,0,0)--++(0,#2,0)--++(-#1,0,0);
374 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
376 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
378 \usepackage[square]{natbib}
379 \bibliographystyle{plainnatmodified}
381 \newcommand{\biburl}[1]{\href{#1}{pdf}}
383 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
385 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
387 \setlength{\cftbeforepartskip}{3ex}
388 \setlength{\cftbeforechapterskip}{1.0ex}
389 \setlength{\cftbeforesectionskip}{0.1ex}
391 %% \setsecnumdepth{subsection}
392 %% \renewcommand{\thesubsection}{\alph{subsection}\,-\hskip -12pt\,}
393 %% \setsecnumformat{\csname the#1\endcsname :}
395 \cftsetindents{part}{0em}{1.8em}
396 \cftsetindents{chapter}{0em}{1.8em}
397 \cftsetindents{section}{1.8em}{2.2em}
399 \setlength{\parindent}{0cm}
400 \setlength{\parskip}{2ex}
402 \setstocksize{15cm}{8cm}
403 \settrimmedsize{\stockheight}{\stockwidth}{*}
404 \setlrmarginsandblock{8pt}{8pt}{*}
405 \setulmarginsandblock{14pt}{26pt}{*}
406 \setheadfoot{14pt}{14pt}
407 \setheaderspaces{*}{*}{*}
408 %% \setlength{\headsep}{0pt}
409 %% \setlength{\headheight}{0pt}
411 %% \newcommand\ignoreme[1]{}
412 %% \setsecheadstyle{\ignoreme}
414 %% \makepagestyle{littlebook}
415 %% \makeoddhead{littlebook}{}{}{}
416 %% \makeevenhead{littlebook}{}{}{}
417 \newcommand{\myfooter}{\footnotesize {\thepage \hskip 0.8em \raisebox{-2pt}{\vline height 8pt} \hskip 0.4em \thelastpage}}
418 %% \makeoddfoot{littlebook}{}{\myfooter}{}
419 %% \makeevenfoot{littlebook}{}{\myfooter}{}
420 \makeoddfoot{plain}{}{\myfooter}{}
421 \makeevenfoot{plain}{}{\myfooter}{}
424 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
426 \renewcommand{\partnamefont}{\centering\sffamily\scshape\Huge}
427 \renewcommand{\partnumfont}{\sffamily\Huge}
428 \renewcommand{\parttitlefont}{\centering\sffamily\scshape\Huge}
429 \renewcommand{\beforepartskip}{\vspace*{\stretch{3}}}
430 \renewcommand{\afterpartskip}{%
431 \vspace*{\stretch{4}}
435 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
437 \makechapterstyle{Tufte}{
438 \renewcommand{\chapterheadstart}{\null \vskip1.5\onelineskip}
439 \renewcommand{\printchaptername}{\large\sffamily\itshape\chaptername}
440 \renewcommand{\printchapternum}{\LARGE\thechapter \\}
441 \renewcommand{\afterchapternum}{}
442 \renewcommand{\printchaptertitle}[1]{
445 \renewcommand{\afterchaptertitle}{
450 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
452 \setsecheadstyle{\sethangfrom{\noindent ##1}\raggedright\sffamily\itshape\Large}
453 \setbeforesecskip{-.9\onelineskip}
454 \setaftersecskip{.75\onelineskip}
456 \setsubsecheadstyle{\sethangfrom{\noindent ##1}\raggedright\sffamily\itshape\large}
457 \setbeforesubsecskip{\onelineskip}
458 \setaftersubsecskip{.65\onelineskip}
460 \setsubsubsecheadstyle{\sethangfrom{\noindent ##1}\raggedright\sffamily\itshape}
461 \setbeforesubsubsecskip{-.5\onelineskip}
462 \setaftersubsubsecskip{.1\onelineskip}
464 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
466 \captiontitlefont{\itshape\small}
467 \captionnamefont{\small}
468 \newcommand{\likecaption}{\color{black}\itshape\small}
472 \checkandfixthelayout
474 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
476 \newcounter{nbdrafts}
477 \setcounter{nbdrafts}{0}
479 \newcommand{\checknbdrafts}{
480 \ifnum \thenbdrafts > 0
481 \@latex@warning@no@line{*WARNING* The document contains \thenbdrafts \space draft note(s)}
483 \newcommand{\todo}[1]{\addtocounter{nbdrafts}{1}{\color{red} #1}}
485 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
486 \definecolor{paper}{rgb}{0.95,0.95,0.95}
487 \definecolor{math}{rgb}{0.0,0.5,0.0}
488 %\definecolor{links}{rgb}{0.0,0.2,0.5}
489 \definecolor{links}{rgb}{0.0,0.2,0.85}
490 %\definecolor{hlcolor}{rgb}{0.8,1.0,0.85}
492 \definecolor{blue}{rgb}{0.3,0.5,0.85}
493 \definecolor{red}{rgb}{0.65,0.0,0.0}
494 \definecolor{green}{rgb}{0.0,0.50,0.0}
495 \definecolor{dimmed}{rgb}{0.8,0.8,0.8}
496 \definecolor{orange}{rgb}{1.0,0.75,0.0}
498 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
499 % Pretty underline, taken from
500 % https://tex.stackexchange.com/questions/36894/underline-omitting-the-descenders
508 \cs_new:Npn \white_text:n #1
510 \fp_set:Nn \l_tmpa_fp {#1 * .01}
511 \llap{\textcolor{white}{\the\SOUL@syllable}\hspace{\fp_to_decimal:N \l_tmpa_fp em}}
512 \llap{\textcolor{white}{\the\SOUL@syllable}\hspace{-\fp_to_decimal:N \l_tmpa_fp em}}
514 \NewDocumentCommand{\whiten}{ m }
516 \int_step_function:nnnN {1}{1}{#1} \white_text:n
520 \NewDocumentCommand{ \prettyul }{ D<>{5} O{0.2ex} O{0.1ex} +m } {%
523 \def\SOUL@uleverysyllable{%
524 \setbox0=\hbox{\the\SOUL@syllable}%
526 \SOUL@ulunderline{\phantom{\the\SOUL@syllable}}%
530 \SOUL@setkern\SOUL@charkern
535 \SOUL@setkern\SOUL@charkern
545 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
548 \usepackage{xcolor, soul}
550 \definecolor{hlcolor}{rgb}{1.0,1.0,0.5}
552 %% \definecolor{ulcolor}{rgb}{0.65,0.65,0.65}
553 %% \setulcolor{ulcolor}
555 %% \index{Attention Layer@\hypertarget{Attention Layer.ind}{}Attention Layer}
556 %% \href{\#Attention Layer.ind}%
558 %% \newcommand{\keyterm}[2][]{%
559 %% \ifthenelse{\equal{#1}{}}
560 %% {\prettyul[2pt]{#2}\linkedindex{#2}}
561 %% {\prettyul[2pt]{#2}\linkedindex{#1}}%
564 \newcommand{\keytermold}[2][]{%
565 % \BeginAccSupp{method=plain,ActualText={#2}}%
566 \ifthenelse{\equal{#1}{}}
567 {\prettyul[2pt]{#2}\index{#2}}
568 {\prettyul[2pt]{#2}\index{#1}}%
574 \newcommand{\keyterm}[2][]{%
576 %% \prettyul[2pt]{#2}%
577 \ifthenelse{\equal{#1}{}}{\index{#2}}{\index{#1}}%
580 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
582 \newcommand{\gizmo}{%
583 \begin{tikzpicture}[scale=0.25]
584 \draw[draw=none,fill=red] (0,0) rectangle ++(1,1);
585 \draw[draw=none,fill=blue] (1,0) rectangle ++(1,1);
586 \draw[draw=none,fill=blue] (0,1) rectangle ++(1,1);
587 \draw[draw=none,fill=red] (1,1) rectangle ++(1,1);
591 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%