% -*- mode: latex; mode: reftex; mode: auto-fill; mode: flyspell; -*-

% Any copyright is dedicated to the Public Domain.
% https://creativecommons.org/publicdomain/zero/1.0/

% Written by Francois Fleuret <francois@fleuret.org>

\documentclass[c,8pt]{beamer}

\usepackage{tikz}
\newcommand{\transpose}{^{\top}}
\def\softmax{\operatorname{softmax}}

\setbeamertemplate{navigation symbols}{}

\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]

Given a query sequence $Q$, a key sequence $K$, and a value sequence
$V$, compute an attention matrix $A$ by matching $Q$s to $K$s, and
weight $V$ with it to get $Y$.

\medskip

\[
\uncover<2,4,6->{
%  A_{i,j} = \softmax \left( \frac{Q_i \cdot K_j}{\sqrt{d}} \right)
  A_i = \softmax \left( \frac{Q_i \, K\transpose}{\sqrt{d}} \right)
}
%
\quad \quad \quad
%
\uncover<3,5->{
  Y_i = A_i V
}
\]

\medskip

\makebox[\textwidth][c]{
\begin{tikzpicture}

  \node[xscale=0.5,yslant=0.5] (V) at (-2, 2.35) {
    \begin{tikzpicture}
      \draw[fill=green!20] (0, 0) rectangle (4, 1.4);
      \uncover<3,5>{\draw[fill=yellow] (0, 0) rectangle (4, 1.4);}
      \foreach \x in { 0.2, 0.4, ..., 3.8 } \draw (\x, 0) -- ++(0, 1.4);
      %% \foreach \y in { 0.0, 0.2, ..., 1.4 } \draw (0, \y) -- ++(4, 0);
    \end{tikzpicture}
  };

  \node[yscale=0.5,xslant=0.5] (A) at (0.5, 1.6) {
    \begin{tikzpicture}
      \draw (0, 0) rectangle ++(3, 4);
      %% \uncover<4->{\draw[fill=green!20] (0, 0) rectangle ++(0.2, 4);}
      %% \uncover<6->{\draw[fill=green!20] (0.2, 0) rectangle ++(0.2, 4);}
    \end{tikzpicture}
  };

  \uncover<2-3>{
  \node[xscale=0.5,yslant=0.5] (a1) at (-0.9, 2.1) {
    \begin{tikzpicture}
      \draw[draw=none] (0, 0) rectangle (4, 1);
      \foreach \x/\y in {
        0.00/0.03, 0.20/0.04, 0.40/0.07, 0.60/0.35, 0.80/0.52,
        1.00/1.00, 1.20/0.82, 1.40/0.25, 1.60/0.08, 1.80/0.03,
        2.00/0.15, 2.20/0.24, 2.40/0.70, 2.60/0.05, 2.80/0.03,
        3.00/0.03, 3.20/0.03, 3.40/0.00, 3.60/0.03, 3.80/0.00
      }{
        \uncover<2>{\draw[black,fill=orange] (\x, 0) rectangle ++(0.2, \y);}
        \uncover<3>{\draw[black,fill=yellow] (\x, 0) rectangle ++(0.2, \y);}
      };
    \end{tikzpicture}
  };
  }

  \uncover<4-5>{
  \node[xscale=0.5,yslant=0.5] (a2) at (-0.7, 2.1) {
    \begin{tikzpicture}
      \draw[draw=none] (0, 0) rectangle (4, 1);
      \foreach \x/\y in {
        0.00/0.03, 0.20/0.04, 0.40/0.07, 0.60/0.03, 0.80/0.03,
        1.00/0.05, 1.20/0.02, 1.40/0.08, 1.60/0.35, 1.80/0.85,
        2.00/0.05, 2.20/0.04, 2.40/0.03, 2.60/0.05, 2.80/0.03,
        3.00/0.03, 3.20/0.03, 3.40/0.00, 3.60/0.03, 3.80/0.00
      }{
        \uncover<4>{\draw[black,fill=orange] (\x, 0) rectangle ++(0.2, \y);}
        \uncover<5>{\draw[black,fill=yellow] (\x, 0) rectangle ++(0.2, \y);}
      };
    \end{tikzpicture}
  };
  }

  \node (Q) at (-0.5, -0.05) {
    \begin{tikzpicture}
      \draw[fill=green!20] (0, 0) rectangle (3, 1.0);
      \foreach \x in { 0.2, 0.4, ..., 2.8 } \draw (\x, 0) -- ++(0, 1.0);
      \uncover<2>{\draw[fill=yellow] (0.0, 0) rectangle ++(0.2, 1);}
      \uncover<4>{\draw[fill=yellow] (0.2, 0) rectangle ++(0.2, 1);}
      %% \foreach \y in { 0.0, 0.2, ..., 1.0 } \draw (0, \y) -- ++(3, 0);
    \end{tikzpicture}
    };

  \node (Y) at (1.5, 3.45) {
    \begin{tikzpicture}
      \uncover<3>{\draw[fill=orange] (0.0, 0) rectangle ++(0.2, 1.4);}
      \uncover<4->{\draw[fill=green!20] (0.0, 0) rectangle ++(0.2, 1.4);}
      \uncover<6->{\draw[fill=green!20] (0.0, 0) rectangle ++(3, 1.4);}
      \uncover<5>{\draw[fill=orange] (0.2, 0) rectangle ++(0.2, 1.4);}
      \draw (0, 0) rectangle (3, 1.4);
      \foreach \x in { 0.2, 0.4, ..., 2.8 } \draw (\x, 0) -- ++(0, 1.4);
      %% \foreach \y in { 0.0, 0.2, ..., 1.4 } \draw (0, \y) -- ++(3, 0);
    \end{tikzpicture}
    };

  \node[xscale=0.5,yslant=0.5] (K) at (3, 1.1) {
    \begin{tikzpicture}
      \draw[fill=green!20] (0, 0) rectangle (4, 1);
      \uncover<2,4>{\draw[fill=yellow] (0, 0) rectangle (4, 1);}
      \foreach \x in { 0.2, 0.4, ..., 3.8 } \draw (\x, 0) -- ++(0, 1);
      %% \foreach \y in { 0.0, 0.2, ..., 1.0 } \draw (0, \y) -- ++(4, 0);
    \end{tikzpicture}
  };

  \node[left of=V,xshift=0.5cm,yshift=0.7cm] (Vl) {$V$};
  \node[left of=Q,xshift=-0.8cm] (Ql) {$Q$};
  \node (Al) at (A) {$A$};
  \node[right of=K,xshift=-0.6cm,yshift=-0.6cm] (Kl) {$K$};
  \node[right of=Y,xshift=0.8cm] (Yl) {$Y$};

  %  \uncover<1>{\draw[<->] (2, 0) -- ++ (0, 1) node[midway,right]{$d$};}

\end{tikzpicture}
}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]

A standard attention layer takes as input two sequences $X$ and $X'$
and computes
%
\begin{align*}
K & = W^K X \\
V & = W^V X \\
Q & = w^Q X' \\
Y & = \underbrace{\softmax_{row} \left( \frac{Q K\transpose}{\sqrt{d}} \right)}_{A} V
\end{align*}

When $X = X'$, this is \textbf{self attention}, otherwise \textbf{cross
  attention.}

\pause

\bigskip

Several such processes can be combined in which case $Y$ is the
concatenation of the separate results. This is \textbf{multi-head
  attention}.

\end{frame}


\end{document}
