\documentclass[10pt]{article}

\usepackage{systeme}

\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
%\usepackage{palatino}
\usepackage{setdeck}

\usepackage{xcolor}
\usepackage{tcolorbox}

%\usepackage[textheight=8.5in, margin=1.6in]{geometry}
\usepackage[total={6in,8in}]{geometry}

\usepackage{tikz}
\usetikzlibrary{cd}
\tikzset{
      labr/.style={anchor=north, rotate=90, inner sep=.7mm}
    }
\tikzset{
      labl/.style={anchor=south, rotate=90, inner sep=.7mm}
    }

%\usepackage{pgfplots}
\tikzstyle{ball} = [circle,shading=ball, ball color=black,
    minimum size=1mm,inner sep=1.3pt]
%\usetikzlibrary{arrows.meta}
%\tikzset{>={Latex[length=2mm,width=1.5mm]}}

%\usepackage{graphicx}

\usepackage{needspace}  % <-- to prevent bad pagebreak over environment titles
% Use:  \Needspace{3\baselineskip} before opening an environment

\usepackage[shortlabels]{enumitem}
\setlist[enumerate]{leftmargin=*,label=(\alph*)}

%\usepackage[colorlinks=true,citecolor=black,linkcolor=black,urlcolor=blue]{hyperref}

\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\sn}{\mathfrak{S}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\ve}{\varepsilon}
\newcommand{\ips}[2]{\langle{#1}, {#2}\rangle}
\DeclareMathOperator{\tr}{tr}
\DeclareMathOperator{\Span}{\mathrm{Span}}
\DeclareMathOperator{\rk}{\mathrm{rank}}
\DeclareMathOperator{\rspace}{\mathrm{rowspace}}
\DeclareMathOperator{\cspace}{\mathrm{colspace}}
\DeclareMathOperator{\trace}{\mathrm{tr}}
\DeclareMathOperator{\im}{\mathrm{im}}
\DeclareMathOperator{\Sym}{\mathrm{Sym}}
\DeclareMathOperator{\Hom}{\mathrm{Hom}}
\DeclareMathOperator{\diag}{\mathrm{diag}}
\DeclareMathOperator{\glb}{\mathrm{glb}}
\DeclareMathOperator{\lub}{\mathrm{lub}}

\parskip = 0.05in
\parindent = 0.0in

%\pagestyle{empty}

\allowdisplaybreaks

\begin{document}
\centerline{Math 201 lecture for Wednesday, Week 12}
\bigskip

\centerline{\textbf{Orthogonal complements and projections}}  
\bigskip
 
\textbf{Definition.} The \emph{direct sum} of vector spaces~$U$ and~$W$ over a
field~$F$ is the set
\[
  U\oplus W=\left\{ (u,w):u\in U\ \text{and}\ w\in W \right\}
\]
with scalar multiplication and vector addition defined by
\[
  \lambda(u,w)=(\lambda u,\lambda w)\quad\text{and}\quad
  (u,w)+(u',w')=(u+u',w+w'),
\]
for all~$u,u'\in U$,~$w,w'\in W$, and~$\lambda\in F$.
\medskip

\textbf{Proposition.}  Let~$U$ and~$W$ be subspaces of a vector space~$V$
over~$F$ such that: (i) the union of~$U$ and~$W$ spans~$V$, and (ii) ~$U\cap
W=\left\{ 0 \right\}$.  Then there is an isomorphism
\begin{align*}
  U\oplus W&\to V\\
  (u,w)&\mapsto u+w.
\end{align*}
Thus, every element of~$V$ has a unique expression of the form~$u+w$
with~$u\in U$ and~$w\in W$.
\begin{proof}
 Easy exercise.
\end{proof}

\textbf{Remark.} In the case of the Proposition, we says that~$V$ is the
\emph{internal direct sum} of~$U$ and~$W$ and abuse notation by simply
writing~$V=U\oplus W$.  The direct sum as we first defined it is sometimes
called the \emph{external direct sum} of~$U$ and~$W$.
\bigskip

For the rest of this lecture, let~$(V,\ips{\ }{\ })$ be an inner product space over~$F=\R$ or~$\C$.
\medskip

\textbf{Definition.} Let~$S\subseteq V$ be nonempty.  The \emph{orthogonal
complement} of~$S$ is
\[
  S^{\perp}=\left\{ x\in V:\ips{x}{y}=0\ \text{for all~$y\in S$} \right\}.
\]
\medskip

\textbf{Exercise.} Show that~$S^{\perp}$ is a subspace of~$V$.
\medskip

\textbf{Example.} Consider~$\R^{3}$ with the standard inner product, and
let~$S=\left\{(a,b,c)\right\}$. So~$S$ consists of the single
vector~$(a,b,c)\in\R^{3}$.  Then
\[
  S^{\perp}=\{(x,y,z)\in\R^{3}: (x,y,z)\cdot(a,b,c)=0\}
=\{(x,y,z)\in\R^{3}: ax+by+cz=0\},
\]
a plane in~$\R^{3}$ defined by the equation~$ax+by+cz=0$.
\medskip

\textbf{Proposition.}  Suppose~$\dim V=n$ and~$S=\left\{ v_1,\dots,v_k \right\}$ is
an orthonormal subset of~$V$.
\begin{enumerate}
  \item\label{12wedit1} $S$ can be extended to an orthonormal basis $\left\{
    v_1,\dots,v_k,v_{k+1},\dots,v_n \right\}$ for~$V$.
  \item\label{12wedit2}  If~$W=\Span S$, then $S'=\left\{ v_{k+1},\dots,v_n \right\}$ is an
    orthonormal basis for~$W^{\perp}$.
  \item\label{12wedit3} If~$W\subseteq V$ is any subspace, then 
    \[
      \dim W + \dim W^{\perp}=\dim V=n.
    \]
  \item\label{12wedit4} If~$W\subseteq V$ is any subspace, then $(W^{\perp})^{\perp}=W$.
\end{enumerate}

\begin{proof}
  \ref{12wedit1} To prove part~\ref{12wedit1}, extend~$S$ to a basis $\left\{
  v_1,\dots,v_k,w_{k+1},\dots,w_n \right\}$ for~$V$, then apply Gram-Schmidt.
  
  \ref{12wedit2} The set~$S'=\left\{ v_{k+1},\dots,v_n \right\}$ is linearly independent
  since it's a subset of a basis.  Since~$\left\{ v_1,\dots,v_n \right\}$ is
  orthonormal, and~$W=\Span \left\{ v_1,\dots,v_k \right\}$, we have
  $S'\subseteq W^{\perp}$.  Therefore,~$\Span S'\subseteq W^{\perp}$.  For the
  opposite inclusion, take~$x\in W^{\perp}$.  Then since $\left\{ v_1,\dots,v_n
  \right\}$ is orthonormal, we have
  \[
    x
    =\sum_{i=1}^n\ips{x}{v_i}\,v_i=
    \sum_{i=k+1}^n\ips{x}{v_i}\,v_i
    \in  \Span S'.
  \]
  
  \ref{12wedit3} If~$W\subseteq V$ is any subspace, choose an orthonormal basis $\left\{
  v_1,\dots, v_k \right\}$ for~$W$.  Then apply parts~\ref{12wedit1} and~\ref{12wedit2}.
  
  \ref{12wedit4} It's clear that~$W\subseteq (W^{\perp})^{\perp}$ since
  \[
    (W^{\perp})^{\perp}=\left\{ x\in V:\ips{x}{y}=0\ \text{for all~$y\in
    W^{\perp}$} \right\}.
  \]
  Then, by part~\ref{12wedit3},
  \[
    \dim (W^{\perp})^{\perp}=n-\dim W^{\perp}=\dim W.
  \]
  Hence,~$W= (W^{\perp})^{\perp}$.
\end{proof}

\textbf{Proposition.} Let~$W$ be a finite-dimensional subspace of~$V$.  Then
\[
  V = W\oplus W^{\perp}.
\]
In other words, for each~$y\in V$, there exist unique~$u\in W$ and~$z\in
W^{\perp}$ such that
\[
  y=u+z.
\]
We define~$u$ to be the \emph{orthogonal projection} of~$y$
onto~$W$. 

If $u_1,\dots,u_k$ is an orthonormal basis for~$W$, then
\[
  u=\sum_{i=1}^k\ips{y}{u_i}\,u_i.
\]
\begin{proof}
 By Gram-Schmidt, there exists an orthonormal basis $u_1,\dots,u_k$
  for~$W$.  Define $u=\sum_{i=1}^k\ips{y}{u_i}\,u_i$ and~$z=y-u$.  Then~$u\in W$
  and~$y=u+z$.  Further,~$z\in W^{\perp}$ since for each~$j=1,\dots,k$, we have
  \begin{align*}
    \ips{z}{u_j}
    &=\ips{y-u}{u_j}\\
    &=\ips{y}{u_j}-\ips{\textstyle\sum_{i=1}^k\ips{y}{u_i}\,u_i}{u_j}\\
    &=\ips{y}{u_j}-\textstyle\sum_{i=1}^k\ips{y}{u_i}\,\ips{u_i}{u_j}\\
    &=\ips{y}{u_j}-\ips{y}{u_j}\,\ips{u_j}{u_j}\\
    &=\ips{y}{u_j}-\ips{y}{u_j}\\
    &=0.
  \end{align*}
  For uniqueness, suppose there exist~$u'\in W$ and~$z'\in W^{\perp}$ such that
  \[
    y = u+z = u'+z'.
  \]
  Then $u-u'=z'-z\in W\cap W^{\perp}=\left\{ 0 \right\}$.  Thus,~$u=u'$
  and~$z=z'$.  (The reason~$W\cap W^{\perp}=\left\{ 0 \right\}$ is as follows:
    if~$x\in W$, then we saw last time that~$x=\sum_{i=1}^k\ips{x}{u_i}\,u_i$.  If
    it is also the case that~$x\in W^{\perp}$, then~$\ips{x}{u_i}=0$ for~$i=1,\dots,k$
    since each~$u_i$ is in~$W$.  Hence,~$x=0$.)
\end{proof}

\textbf{Corollary.} The orthogonal projection~$u$ of~$y$ onto~$W$ is the closest
vector in~$W$ to~$y$:
\[
\lVert y-u\rVert \leq \lVert y-w\rVert
\]
for all~$w\in W$ with equality if and only if~$w=u$.
\begin{proof}
  Write~$y=u+z$ with~$u\in W$ and~$z\in W^{\perp}$, and
  let~$w\in W$.  Then~$u-w\in W$ and~$y-u\in W^{\perp}$.  So~$u-w$ and~$z=y-u$ are
  perpendicular.  By the Pythagorean theorem,
  \begin{align*}
    \lVert y-w\rVert^2
    &=\lVert (u+z)-w\rVert^2\\
    &=\lVert (u-w)+z\rVert^2\\
    &=\lVert (u-w)\rVert^2+\lVert z\rVert^2\\
    &\geq\lVert z\rVert^2\\
    &=\lVert y-u\rVert^2.
  \end{align*}
  Equality occurs above if and only if~$\lVert u-w\rVert^2=0$, i.e., if and only
  if~$u=w$.
\end{proof}

\textbf{Example.} Let~$V=\R^3$ with the standard inner product, and let's consider
orthogonal projection onto the~$xy$-plane.  An orthonormal basis for
the~$xy$-plane is~$\left\{ e_1,e_2 \right\}$.  The projection of a
point~$u=(x,y,z)\in\R^3$ is given by
\[
  u=((x,y,z)\cdot e_1)e_1+((x,y,z)\cdot e_2)e_2=x\,e_1+y\,e_2=(x,y,0). 
\]
The distance of~$(x,y,z)$ to the~$xy$-plane is
\[
  \lVert (x,y,z)-u\rVert=\lVert (0,0,z)\rVert=|z|.
\]

\textbf{Application.}  Consider the vector space~$V$ of integrable
functions~$f\colon[0,2\pi]\to\R$ with inner product
\[
  \langle f,g\rangle:=\frac{1}{\pi}\int_{0}^{2\pi}f(t)g(t)\,dt.
\]
Thus, the distance between~$f,g\in V$ is
\[
  \lVert f-g\rVert
  =\sqrt{\frac{1}{\pi}\int_{0}^{2\pi}(f(t)-g(t))^2\,dt},
\]
which will be small if~$f(t)\approx g(t)$ for~$t\in[0,2\pi]$.

One may check that
$S_n:=\left\{\frac{1}{\sqrt{2}},\cos(x),\sin(x),\cos(2x),\sin(2x),\ldots,\cos(nx),\sin(nx)\right\}$
is an orthonormal subset.  Given any integrable~$f\in V$, the orthogonal
projection of~$f$ to the subspace spanned by~$S_n$ gives the best approximation
of the function using sines and cosines of frequencies~$\frac{j}{2\pi}$
for~$j=0,\ldots,n$.  Write the projection of~$f$ to $\Span(S_{n})$ as
\[
  \mathrm{proj}_{\Span(S_{n})}(f)(x) = \alpha\cdot\frac{1}{\sqrt{2}} + \sum_{i=1}^{n}
  \beta_{i}\cos(ix)+\sum_{i=1}^{n} \gamma_{i}\sin(ix),
\]
Since~$S_{n}$ is orthonormal, we may find the coefficients by taking inner
products:
\begin{align*}
  \alpha &= \langle f,1/\sqrt{2}\rangle =\frac{1}{\pi}\int_{0}^{2\pi}\frac{f(t)}{\sqrt{2}}\,dt\\[5pt]
  \beta_{i} &= \langle f,\cos(ix)\rangle =\frac{1}{\pi}\int_{0}^{2\pi}f(t)\cos(ix)\,dt\\[5pt]
  \gamma_{i} &= \langle f,\sin(ix)\rangle =\frac{1}{\pi}\int_{0}^{2\pi}f(t)\sin(ix)\,dt.
\end{align*}
For instance, consider the function~$f(x)=x$ for~$x\in[0,2\pi]$.  We find
\[
  \alpha = \frac{1}{\pi}\int_{0}^{2\pi}\frac{t}{\sqrt{2}}\,dt = \sqrt{2}\pi.
\]
Integrating by parts, we find
\[
  \beta_i=\frac{1}{\pi}\int_{0}^{2\pi}t\cos(it)\,dt
  =\frac{1}{\pi}\left(\frac{t\sin(it)}{i} + \frac{\cos(it)}{i^{2}}\right|_{0}^{2\pi}
  = 0
\]
and
\[
  \gamma_i=\frac{1}{\pi}\int_{0}^{2\pi}t\sin(it)\,dt
  =\frac{1}{\pi}\left(-\frac{t\cos(it)}{i} + \frac{\sin(it)}{i^{2}}\right|_{0}^{2\pi}
    =-\frac{2}{i}. 
\]
Thus,
\[
  \mathrm{proj}_{\Span(S_{n})}(f)(x) = \sqrt{2}\pi\cdot
  \frac{1}{\sqrt{2}}-
  \sum_{i=1}^{n}\frac{2}{i\pi}\sin(ix)
  = \pi - \frac{2}{\pi}\sum_{i=1}^{n} \frac{\sin(ix)}{i}.
\]
See the next page to compare the graph of~$f$ with the graphs of these
projections for various~$n$.
\newpage

The plot of $\mathrm{proj}_{\Span(S_{n})}(f)$ versus the plot of~$f(x)=x$ for
$n=1,2,$ and~$10$.
\begin{center}
  \includegraphics[height=2in]{images/12wed1.png}
\end{center}
\begin{center}
  \includegraphics[height=2in]{images/12wed2.png}
\end{center}
\begin{center}
  \includegraphics[height=2in]{images/12wed10.png}
\end{center}
\end{document}  
