\documentclass[10pt]{article}

\usepackage{systeme}

\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
%\usepackage{palatino}
\usepackage{setdeck}

\usepackage{xcolor}
\usepackage{tcolorbox}

%\usepackage[textheight=8.5in, margin=1.6in]{geometry}
\usepackage[total={6in,8in}]{geometry}

\usepackage{tikz}
\usetikzlibrary{cd}
\tikzset{
      labr/.style={anchor=north, rotate=90, inner sep=.7mm}
    }
\tikzset{
      labl/.style={anchor=south, rotate=90, inner sep=.7mm}
    }

%\usepackage{pgfplots}
\tikzstyle{ball} = [circle,shading=ball, ball color=black,
    minimum size=1mm,inner sep=1.3pt]
%\usetikzlibrary{arrows.meta}
%\tikzset{>={Latex[length=2mm,width=1.5mm]}}

%\usepackage{graphicx}

\usepackage{needspace}  % <-- to prevent bad pagebreak over environment titles
% Use:  \Needspace{3\baselineskip} before opening an environment

\usepackage[shortlabels]{enumitem}
\setlist[enumerate]{leftmargin=*,label=(\alph*)}

%\usepackage[colorlinks=true,citecolor=black,linkcolor=black,urlcolor=blue]{hyperref}

\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\sn}{\mathfrak{S}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\ve}{\varepsilon}
\newcommand{\ips}[2]{\langle{#1}, {#2}\rangle}
\DeclareMathOperator{\tr}{tr}
\DeclareMathOperator{\Span}{\mathrm{Span}}
\DeclareMathOperator{\rk}{\mathrm{rank}}
\DeclareMathOperator{\rspace}{\mathrm{rowspace}}
\DeclareMathOperator{\cspace}{\mathrm{colspace}}
\DeclareMathOperator{\trace}{\mathrm{tr}}
\DeclareMathOperator{\im}{\mathrm{im}}
\DeclareMathOperator{\Sym}{\mathrm{Sym}}
\DeclareMathOperator{\Hom}{\mathrm{Hom}}
\DeclareMathOperator{\diag}{\mathrm{diag}}
\DeclareMathOperator{\glb}{\mathrm{glb}}
\DeclareMathOperator{\lub}{\mathrm{lub}}

\parskip = 0.05in
\parindent = 0.0in

%\pagestyle{empty}

\allowdisplaybreaks

\begin{document}
\centerline{Math 201 lecture for Monday, Week 12}
\bigskip

\centerline{\textbf{Gram-Schmidt}}  
\bigskip

Let~$(V,\langle\ ,\ \rangle)$ be an inner product space over~$F=\R$ or~$\C$.
\medskip

\textbf{Definition.} Let~$S\subseteq V$.  Then~$S$ is an \emph{orthogonal}
subset of~$V$
if~$\ips{u}{v}=0$ for all~$u,v\in S$ with~$u\neq v$. If~$S$ is an orthogonal
subset of~$V$ and~$\lVert u\rVert=1$ for all~$u\in S$, then~$S$ is an
\emph{orthonormal} subset of~$V$.
\medskip

\textbf{Examples.} 
\begin{itemize}
  \item The standard basis~$e_1,\dots,e_n$ for~$F^n$ is orthonormal with respect
    to the standard inner product on~$F^n$.
  \item $\left\{ \frac{1}{\sqrt{2}}(1,1),\frac{1}{\sqrt{2}}(1,-1) \right\}$ is
    orthonormal with respect to the standard inner product on~$\R^2$.
\end{itemize} 
\medskip

\textbf{Proposition.} Let ~$S=\left\{ v_1,\dots,v_k \right\}$ be an orthogonal set
of nonzero vectors in~$V$, and let~$y\in\Span S$.  Then
\[
  y = \sum_{j=1}^k\frac{\ips{y}{v_j}}{\ips{v_j}{v_j}}\,v_j=
  \sum_{j=1}^k\frac{\ips{y}{v_j}}{\lVert v_j\rVert^2}\,v_j.
\]
Note that the coefficients are the components of~$y$ along each~$v_j$.

\begin{proof}
  Say~$y=\sum_{i=1}^ka_iv_i$.  Then for~$j=1,\dots,k$,
  \[
    \ips{y}{v_j}=\ips{\textstyle\sum_{i=1}^ka_iv_i}{v_j}=
    \textstyle\sum_{i=1}^ka_i\ips{v_i}{v_j}=a_j\ips{v_j}{v_j},
  \]
  since~$\ips{v_i}{v_j}=0$ for~$i\neq j$.  Hence,
  \[
    a_j=\frac{\ips{y}{v_j}}{\ips{v_j}{v_j}}=\frac{\ips{y}{v_j}}{\lVert
    v_j\rVert^2},
  \]
  the component of~$y$ along~$v_j$.
\end{proof}

\textbf{Corollary 1.} If~$S=\left\{ v_1,\dots,v_k \right\}$ is orthonormal and
$y\in\Span S$, then
\[
  y=\sum_{i=1}^k\ips{y}{v_j}v_i.
\]
\medskip

\textbf{Corollary 2.} Is~$S=\left\{ v_1,\dots,v_k \right\}$ is an orthogonal set
of nonzero vectors in~$V$ then~$S$ is linearly independent.

\begin{proof}
If~$\sum_{i=1}^ka_iv_i=0$, then for each~$j=1,\dots,k$,
  \[
  0=\ips{0}{v_j}=\ips{\textstyle\sum_{i=1}^k a_iv_i}{v_j}=a_j\ips{v_j}{v_j}.
  \]
  Since~$v_j\neq0$ and $\ips{\ }{\ }$ is positive-definite, we have
  $\ips{v_j}{v_j}\neq0$.  Hence,~$a_j=0$ for $j=1,\dots,k$.
\end{proof}

\textbf{Example.} Consider~$\R^2$ with the standard inner product, and let
\[
  u=\frac{1}{\sqrt{2}}(1,1)\quad\text{and}\quad v=\frac{1}{\sqrt{2}}(1,-1).
\]
Then~$\beta=\left\{ u,v \right\}$ gives an orthonormal ordered basis for~$\R^2$.  What are the
coordinates of~$y=(4,1)$ with respect to that basis?
\begin{center}
  \begin{tikzpicture}
    \draw[->] (-1,0)--(5,0);
    \draw[->] (0,-1)--(0,2);
    \draw[blue,thick,->] (0,0)--(0.7071,0.7071);
    \node at (0.6,1) (u) {$u$};
    \draw[blue,thick,->] (0,0)--(0.7071,-0.7071);
    \node at (0.5,-1) (v) {$v$};
    \node[ball,label={right:$(4,1)$}] at (4,1) {};
    \draw[densely dashed,->] (0.7071,0.7071)--(2.5,2.5);
    \draw[densely dashed,->] (0.7071,-0.7071)--(1.5,-1.5);
    \draw[thick, dotted] (2.5,2.5)--(4,1);
    \draw[thick, dotted] (1.5,-1.5)--(4,1);
    \node at (2,2.7) {$\frac{5}{\sqrt{2}}u$};
    \node at (1.2,-1.8) {$\frac{3}{\sqrt{2}}v$};
  \end{tikzpicture}
\end{center}
Answer:
\begin{align*}
  y
  &=\ips{y}{u}u+\ips{y}{v}v\\[8pt]
  &=(4,1)\cdot\left(\frac{1}{\sqrt{2}}(1,1)\right)u+(4,1)\left( \frac{1}{\sqrt{2}} (1,-1)\right)v\\[8pt]
  &=\frac{5}{\sqrt{2}}\,u+\frac{3}{\sqrt{2}}\,v.
\end{align*}
Check:
\[
  \frac{5}{\sqrt{2}}\left( \frac{1}{\sqrt{2}}(1,1) \right)
  +\frac{3}{\sqrt{2}}\left( \frac{1}{\sqrt{2}}(1,-1) \right)=
  \frac{5}{2}\,(1,1)+\frac{3}{2}(1,-1)=(4,1).
\]
\medskip

\textbf{Gram-Schmidt.} Given vectors~$w_1,w_2\in V$, we'd like to compute
orthogonal vectors~$v_1,v_2$ such that
\[
  \Span\left\{ w_1,w_2 \right\}=\Span\left\{ v_1,v_2 \right\}.
\]
To do that, let~$v_1=w_1$, then ``straighten out''~$w_2$ to create~$v_2$:
\begin{center}
  \begin{tikzpicture}
    \draw[->] (0,0)--(3.5,0);
    \draw[->] (0,0)--(2.0,0);
    \node at (3.5,-0.3) {$w_1=v_1$};
    \draw[->] (0,0)--(2,2);
    \node at (1.7,2.2) {$w_2$};
    \draw[dashed] (2,0)--(2,2);
    \node at (2.0,-0.3) {$cv_1$};
    \node at (-0.2,-0.2) {$\vec{0}$};
    \node at (3.4,1.0) {$v_2=w_2-cv_1$};
  \end{tikzpicture}
\end{center}
The number~$c$ is the component of~$w_2$ along~$v_1$.  Recall,~$c$ is determined
by requiring~$v_2$ and~$v_1$ to be orthogonal:
\[
  0=\ips{v_2}{v_1}=\ips{w_2-cv_1}{v_1}=\ips{w_2}{v_2}-c\ips{v_1}{v_1}.
\]
Therefore,
\[
  c=\frac{\ips{w_2}{v_1}}{\ips{v_1}{v_1}}
  =\frac{\ips{w_2}{v_1}}{\lVert v_1\rVert^2}.
\]
(We've assumed~$v_1\neq0$.)

The following algorithm generalizes this idea:
\medskip

\textbf{Algorithm.} (Gram-Schmidt orthogonalization) 

{\sc input:} $S=\left\{ w_1,\dots,w_n \right\}$, a linearly independent subset
of~$V$.

Let
\[
  v_1:=w_1.
\]
For $k=2,3,\dots,n$, define~$v_k$ by starting with~$w_k$, then subtracting
off the components of~$w_k$ along the previously found~$v_i:$
\[
  v_k:=w_k-\sum_{i=1}^{k-1}\frac{\ips{w_k}{v_i}}{\lVert v_i\rVert^2}v_i.
\]
{\sc output:} $\displaystyle S'=\left\{ v_1,\dots,v_n \right\}$ an orthogonal set
with~$\Span S'=\Span S$.
\medskip

\ \ \ \ or
\medskip

\textsc{output:} $\displaystyle S''=\left\{ \frac{v_1}{\lVert
v_1\rVert},\dots,\frac{v_n}{\lVert v_n\rVert} \right\}$ an orthonormal set
with~$\Span S'=\Span S$.

\begin{proof}[Proof of validity of the algorithm.] We prove this by induction on~$n$.  The case~$n=1$ is clear. 
  Suppose the algorithm works for some~$n\geq 1$, and let $S=\left\{
  w_1,\dots,w_{n+1}\right\}$ be a linearly independent set.  By induction, running the algorithm
  on the first~$n$ vectors in~$S$ produces orthogonal~$v_1,\dots,v_n$ with
  \[
    \Span\left\{ v_1,\dots,v_n \right\}=\Span\left\{ w_1,\dots,w_n \right\}.
  \]
  Running the algorithm further produces
  \[
    v_{n+1}=w_{n+1}-\sum_{i=1}^{n}\frac{\ips{w_{n+1}}{v_i}}{\lVert v_i\rVert^2}v_i.
  \]
  It cannot be that~$v_{n+1}=0$, since otherwise, the above equation we would
  say
  \[
    w_{n+1}\in\Span\left\{ v_1,\dots,v_n \right\}=\Span\left\{ w_1,\dots,w_n
    \right\},
  \]
  contradicting the assumption of the linear independence of the~$w_i$.
  So~$v_{n+1}\neq 0$.  

  We now check that~$v_{n+1}$ is orthogonal to the previous~$v_i$.
  For~$j=1,\dots,n$, we have
  \begin{align*}
    \ips{v_{n+1}}{v_j}
      &=\bigg\langle w_{n+1}-\sum_{i=1}^{n}\frac{\ips{w_{n+1}}{v_i}}{\lVert v_i\rVert^2}v_i,v_j\bigg\rangle\\[10pt]
      &=\ips{w_{n+1}}{v_j}
      -\sum_{i=1}^{n}\frac{\ips{w_{n+1}}{v_i}}{\lVert v_i\rVert^2}\,\langle v_i,v_j\rangle\\[10pt]
      &=\ips{w_{n+1}}{v_j} -\frac{\ips{w_{n+1}}{v_j}}{\lVert v_j\rVert^2}\,\langle v_j,v_j\rangle\\[10pt]
      &=\ips{w_{n+1}}{v_j} -\ips{w_{n+1}}{v_j}\\[10pt]
      &=0.
  \end{align*}
  We have shown~$\left\{ v_1,\dots,v_{n+1} \right\}$ is an orthogonal set of
  vectors, and we would now like to show that its span is the span of $\left\{
  w_1,\dots,w_{n+1} \right\}$.  First, since $\left\{ v_1,\dots,v_{n+1} \right\}$
  is orthogonal, it's linearly independent.  Next, we have
  \[
    \Span\left\{ v_1,\dots,v_{n+1} \right\}\subseteq
    \Span\left\{ v_1,\dots,v_{n},w_{n+1} \right\}\subseteq
    \Span\left\{ w_1,\dots,w_{n},w_{n+1} \right\}.
  \]
  Since~$\Span\left\{ v_1,\dots,v_{n+1} \right\}$ is an~$(n+1)$-dimensional subspace
  of the~$(n+1)$-dimensional space $\Span\left\{ w_1,\dots,w_{n},w_{n+1}
  \right\}$, these spaces must be equal.
\end{proof}

\textbf{Corollary.} Every nonzero finite-dimensional inner product space has an
orthonormal basis.
\medskip

\textbf{Example.} Let~$V=\R_{\leq 1}[x]$, the space of polynomials of degree at most~$1$ with real
coefficients and with inner product
\[
  \ips{f}{g}=\int_0^1f(t)g(t)\,dt.
\]
Apply Gram-Schmidt to the basis~$\left\{ 1,x \right\}$ to get an orthonormal basis.
Note that~$1$ and~$x$ are not orthogonal:
\[
  \ips{1}{x}=\int_0^1t\,dt=\frac{1}{2}\neq0.
\]
Gram-Schmidt: Start with $v_1=1$, then let
\begin{align*}
  v_2&=x-\frac{\ips{x}{v_1}}{\lVert v_1\rVert^2}\,v_1\\[10pt]
  &= x-\frac{\ips{x}{1}}{\lVert1\rVert^2}\cdot 1\\[10pt]
  &= x-\frac{\int_0^1t\,dt}{\int_0^1 dt}\cdot 1\\[10pt]
  &=x-\frac{1}{2}.
\end{align*}
Check orthogonality:
\[
  \ips{1}{x-1/2}=\int_0^1(t-1/2)\,dt=0.
\]
Now scale~$v_1=1$ and~$v_2=x-1/2$ to create an orthonormal basis:
\begin{align*}
  \lVert v_1\rVert&=\sqrt{\int_0^1\,dt}=1\\
  \lVert v_2\rVert&=\sqrt{\ips{x-1/2}{x-1/2}}\\
  &=\sqrt{\int_0^1(t-1/2)^2\,dt}\\
  &=\sqrt{\int_0^1(t^2-t+1/4)\,dt}\\
  &=\sqrt{1/12}.
\end{align*}
So an orthonormal basis for~$V$ is
\[
\left\{ 1,\sqrt{12}(x-1/2) \right\}.
\]
\end{document}  
