\documentclass[12pt]{article}

\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}

\usepackage{xcolor}

\usepackage{systeme}

%\usepackage[textheight=8.5in, margin=1.6in]{geometry}
\usepackage[total={6in,8in}]{geometry}

\usepackage{tikz}
%\usepackage{pgfplots}
\tikzstyle{ball} = [circle,shading=ball, ball color=black,
    minimum size=1mm,inner sep=1.3pt]
%\usetikzlibrary{arrows.meta}
%\tikzset{>={Latex[length=2mm,width=1.5mm]}}

%\usepackage{graphicx}

\usepackage{needspace}  % <-- to prevent bad pagebreak over environment titles
% Use:  \Needspace{3\baselineskip} before opening an environment

\usepackage{enumitem}
%\setlist[enumerate]{leftmargin=*}

%\usepackage[colorlinks=true,citecolor=black,linkcolor=black,urlcolor=blue]{hyperref}

\newcommand{\N}{\mathbb{N}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\sn}{\mathfrak{S}}
\newcommand{\ve}{\varepsilon}
\newcommand{\ips}[2]{\langle{#1}, {#2}\rangle}
\DeclareMathOperator{\tr}{tr}
\DeclareMathOperator{\Span}{\mathrm{Span}}
\DeclareMathOperator{\rk}{\mathrm{rank}}
\DeclareMathOperator{\trace}{\mathrm{tr}}
\DeclareMathOperator{\im}{\mathrm{im}}
\DeclareMathOperator{\Sym}{\mathrm{Sym}}
\DeclareMathOperator{\Hom}{\mathrm{Hom}}
\DeclareMathOperator{\diag}{\mathrm{diag}}
\DeclareMathOperator{\glb}{\mathrm{glb}}
\DeclareMathOperator{\lub}{\mathrm{lub}}

\parskip = 0.05in
\parindent = 0.0in

%\pagestyle{empty}

\begin{document}
\centerline{Math 201 lecture for Wednesday, Week 3}
\bigskip

\centerline{\bf Linear independence}
\medskip

{\bf Definition.} A set~$S\subset V$ is \emph{linearly dependent} if there exist
distinct\footnote{Note the easily forgotten but necessary word ``distinct'',
here.}~$u_1,\dots,u_n\in S$, for some~$n\geq1$, and scalars~$a_1,\dots,a_n$, not
all zero, such that
\[
  a_1u_1+\dots+a_nu_n=0.
\]
We call the above expression a {\em non-trivial dependence relation} among
the~$u_i$.
\medskip

{\bf Example.}  The empty set is not linearly dependent.
\medskip

{\bf Example.} If~$0\in S$, then~$S$ is linearly dependent.  For
instance,~$1\cdot 0=0$ is a non-trivial dependence relation.  
\medskip

{\bf Example.} Let~$S=\left\{ (1,-1,0),(-1,0,2),(-5,3,4)
\right\}\subset\R^3$.
Is~$S$ linearly dependent?  We look for~$a_1,a_2,a_3\in\R$ such that
\[
  a_1(1,-1,0)+a_2(-1,0,2)+a_3(-5,3,4)=(0,0,0),
\]
i.e., such that
\[
  (a_{1}-a_{2}-5a_{3},-a_{1}+3a_{3},2a_{2}+4a_{3})=(0,0,0).
\]
So we are looking for a solution to the system of linear equations
\[
  \sysdelim..
  \systeme{a_1-a_2-5a_3=0,-a_1+3a_3=0,2a_2+4a_3=0.}
\]
Apply our algorithm:
\begin{align*}
  &\left(\begin{array}{rrr|r}
      1&-1&-5&0\\
      -1&0&3&0\\
      0&2&4&0
  \end{array} \right)
  \xrightarrow{r_2\to r_2+r_1}
  \left(\begin{array}{rrr|r}
      1&-1&-5&0\\
      0&-1&-2&0\\
      0&2&4&0
  \end{array} \right)
  \xrightarrow{r_2\to -r_2}\\[10pt]
  &\left(\begin{array}{rrr|r}
      1&-1&-5&0\\
      0&1&2&0\\
      0&2&4&0
  \end{array} \right)
  \xrightarrow[r_3\to r_3-2r_2]{r_1\to r_1+r_2}
  \left(\begin{array}{rrr|r}
      1&0&-3&0\\
      0&1&2&0\\
      0&0&0&0
  \end{array} \right).
\end{align*}
Converting back to a system of equations and solving for the pivot variables
gives
\[
  a_1=3a_3,\quad a_2=-2a_3,
\]
and~$a_3$ is arbitrary.  Take~$a_3=1$ to get the solution~$a_1=3$, $a_2=-2$,
and~$a_3=1$:
\[
  3(1,-1,0)-2(-1,0,2)+(-5,3,4)=(0,0,0).
\]
Therefore, these vectors are linearly dependent.
\medskip

{\bf Proposition 1.} Let~$S\subseteq V$.  Then~$S$ is linearly dependent if and
only if there exists~$v\in S$ such that~$v$ is a linear combination of vectors
in~$S\setminus\left\{ v \right\}$, i.e., if and only
  if~$v\in\Span(S\setminus\left\{ v \right\})$.
\smallskip

\begin{proof}
  First note that we may assume~$S\neq\emptyset$ since the empty set is not
  linearly dependent.  
  
  ($\Rightarrow$) Suppose~$a_1u_1+\dots+a_nu_n=0$ for 
  distinct~$u_i\in S$ and~$a_i\in F$, not all zero.  Without loss of generality,
  we may assume that~$a_1\neq0$.  In that case, we have
  \[
    u_1=-\frac{a_2}{a_1}u_2-\frac{a_3}{a_1}u_3-\dots-\frac{a_n}{a_1}u_n,
  \]
  expressing~$u_1$ as a linear combination of elements in~$S\setminus\left\{ u_1
  \right\}$.
  Note the special case where~$S=\left\{ 0 \right\}$.  The result still holds in
  that case since~$\left\{ 0 \right\}=\Span(\emptyset)$.  By definition, the empty
  linear combination is~$0$.
  
  ($\Leftarrow$) Say~$v=a_1u_1+\dots+a_n u_n$ with distinct~$u_i\in S\setminus\left\{ v
  \right\}$ and~$v\in S$.  Then
  \[
  a_1u_1+\dots+a_n u_n-v=0
  \]
  shows that~$S$ is linearly dependent.
\end{proof}

\textbf{Definition.} A set~$S\subset V$ is \emph{linearly independent} if it
is not linearly dependent.  This means that for all~$n\geq 1$
and distinct~$u_1,\dots,u_n\in S$, if $a_1u_1+\dots+a_nu_n=0$ for some~$a_i\in F$,
then~$a_1=\dots=a_n=0$.  (In particular, the empty set is linearly independent.)
\medskip

{\bf Remark.} We say there is a \emph{linear relation} among
vectors~$u_1,\dots,u_n$ if there exist $a_i\in F$ such
that~$a_1u_1+\dots+a_nu_n=0$.  The linear relation is \emph{trivial} if
all~$a_i=0$.  Thus, a subset~$S$ of~$V$ is linearly independent if every linear
relation distinct elements of~$S$ is trivial.
\medskip

\textbf{IMPORTANT.}  To prove that a set of (distinct) vectors $S=\{v_1,\ldots,v_k\}$ is
linearly independent start by writing the following:

\begin{quote}
  Suppose that
  \[
    a_1v_1+\cdots+a_kv_k=0
  \]
  for some~$a_1,\ldots,a_{k}\in F$.
\end{quote}

The goal is then to use some knowledge you are given about the vectors
$v_1,\ldots,v_k$ to show that the relation is trivial, i.e.,~$a_i=0$ for all~$i$.

\textbf{AVOID.}  Another way to prove that a set of vectors
$S=\{v_1,\ldots,v_k\}$ is linearly independent is to suppose that some $v_i$ is
a linear combination of the vectors in $S \setminus \{v_i\}$ or to suppose that
there is some nontrivial linear combination of elements in~$S$, and then show a
contradiction arises.  Whenever tempted to give such a proof, check to see if
the standard proof, described just above, would be clearer (as it almost always
will).
\medskip

\textbf{Examples.} 
\begin{itemize}
  \item The set $\left\{ u \right\}$ is linearly independent for any nonzero~$u\in V$:
    if~$\lambda u=0$ for some~$\lambda\neq0$, then scaling by~$1/\lambda$
    would give~$u=0$.  But we are supposing~$u\neq0$.  (Here is a case where the
    indirect proof of independence seems warranted.)
  \item $S=\left\{ (1,-1,0),(-1,0,2),(0,1,1) \right\}\subset\R$ is linearly
    independent.  To see this, we follow the standard proof.  Suppose that
    \[
      a(1,-1,0)+b(-1,0,2)+c(0,1,1)=0,
    \]
    which means
\[
  \sysdelim..
  \systeme{a-b=0,-a+c=0,2b+c=0.}
\]
Apply our algorithm (I'll just show the result of row reduction):
\[
  \left(\begin{array}{rrr|r}
      1&-1&0&0\\
      -1&0&1&0\\
      0&2&1&0
  \end{array} \right)
  \rightsquigarrow
  \left(\begin{array}{rrr|r}
      1&0&0&0\\
      0&1&0&0\\
      0&0&1&0
  \end{array} \right).
\]
Thus, the only solution is~$a=b=c=0$.
\item The set $S=\{1+x,1+x+x^{2}\}\subset P_2(\R)=\R[x]_{\leq 2}$ is linearly
  independent.  To see this, suppose that
  \[
    a(1+x)+b(1+x+x^2)=0
  \]
  for some~$a,b\in\R$.  It follows that
  \[
    (a+b)+(a+b)x+bx^{2}=0,
  \]
  and, therefore, $a+b=0$ (the coefficient the constant term or of the $x$-term)
  and~$b=0$ (the coefficient of the $x^{2}$-term).  It then follows
  that~$a=b=0$.
\end{itemize}
\bigskip

\textbf{Problem (leading to an important algorithm).}  Let
\[
  S = \left((2,0,0),(0,1,0),(2,2,0),(0,3,1),(3,0,1)\right).
\]
Find a linearly independent subset of~$S$ and write the remaining vectors as
linear combinations of vectors in that subset.
\smallskip

\textbf{Solution.} Look for linear relations 
\[
  c_{1}(2,0,0)+c_{2}(0,1,0)+c_{3}(2,2,0)+c_{4}(0,3,1)+c_{5}(3,0,1)=(0,0,0).
\]
Convert the above relation to as system of three homogeneous linear equations in
$c_1,c_2,c_3,c_4,c_5$ and solve:
\[
  \left(\begin{array}{ccccc|c}
    2&0&2&0&3&0\\
    0&1&2&3&0&0\\
    0&0&0&1&1&0
  \end{array}\right)
  \rightsquigarrow
  \left(\begin{array}{ccccc|c}
    1&0&1&0&\frac{3}{2}&0\\
    0&1&2&0&-3&0\\
    0&0&0&1&1&0
  \end{array}\right).
\]
(Note that the first matrix has the vectors in~$S$ as columns.)
So the solution space is
\[
  \left(-c_{3}-\frac{3}{2}c_{5},
  -2c_{3}+3_{5},c_{3},-c_{5},c_{5}:c_{3},c_{5}\in\R \right),
\]
or, in parametric form
\[
  \left\{
    c_{3}\left(\begin{array}{r}
	-1\\
	-2\\
	1\\
	0\\
	0
    \end{array}\right)
    +
    c_{5}\left(\begin{array}{r}
	-\frac{3}{2}\\
	3\\
	0\\
	-1\\
	1
    \end{array}\right)
    : c_{3},c_{5}\in \R
  \right\}.
\]
Let~$T$ be the set of columns in our original matrix with the same indices as
those for the non-free (i.e., pivot or leading) variables in the row-reduced matrix.  In other
words,
\[
  T=
  \left\{
    \left(\begin{array}{r}
	2\\
	0\\
	0
    \end{array}\right),
    \left(\begin{array}{r}
	0\\
	1\\
	0
    \end{array}\right),
    \left(\begin{array}{r}
	0\\
	3\\
	1
    \end{array}\right)
  \right\}.
\]
We claim that~$T$ is linearly independent.  Suppose there is a linear
relation (switching to row notation for convenience):
\[
  a(2,0,0)+b(0,1,0)+c(0,3,1)=0.
\]
To show that~$a=b=c=0$ is the only solution, we convert to a matrix and
row-reduce as usual:
\[
  \left(
  \begin{array}{ccc}
  2 &0  & 0  \\
  0 & 1 & 3 \\
  0 & 0& 1 
  \end{array}
  \right)
  \rightsquigarrow
  \left(
  \begin{array}{ccc}
  1 &0  & 0  \\
  0 & 1 & 0 \\
  0 & 0& 1 
  \end{array}
  \right).
\]
Therefore, we must have~$a=b=c=0$, as claimed.  \textbf{Important:} In fact,
there was no need to do that last computation since we have already done it.
To see that, go back to our original row-reduction
\[
  \left(\begin{array}{ccccc|c}
    2&0&2&0&3&0\\
    0&1&2&3&0&0\\
    0&0&0&1&1&0
  \end{array}\right)
  \rightsquigarrow
  \left(\begin{array}{ccccc|c}
    1&0&1&0&\frac{3}{2}&0\\
    0&1&2&0&-3&0\\
    0&0&0&1&1&0
  \end{array}\right) 
\]
and only pay attention to the first, second, and fourth columns.  So the
verification that~$T$ is linearly independent was secretly guaranteed by its
construction.

It remains to be shown that the remaining columns (those corresponding to the
free variables), i.e., $(2,2,0)$ and $(3,0,1)$, in row notation) are in the span of~$T$. We
have found all solutions to
\begin{equation}\label{eqn:03wed}
  c_{1}(2,0,0)+c_{2}(0,1,0)+c_{3}(2,2,0)+c_{4}(0,3,1)+c_{5}(3,0,1)=(0,0,0)
\end{equation}
and found that~$c_{3}$ and $c_{5}$ are free variables.  To see that~$(2,2,0)$ is
in the span of~$T$, find the solution to our system for
which~$(c_{3},c_{5})=(1,0)$, then solve for~$(2,2,0)$ in~\eqref{eqn:03wed}.  The
solution in this case is
\[
  \left(\begin{array}{c}
      c_{1}\\
      c_{2}\\
      c_{3}\\
      c_{4}\\
      c_{5}
  \end{array}\right)
  =
\left(\begin{array}{r}
      -1\\
      -2\\
      1\\
      0\\
      0
  \end{array}\right)
\]
Therefore,
\[
  -(2,0,0)-2(0,1,0)+1\cdot(2,2,0)+0\cdot(0,3,1)+0\cdot(3,0,1)=(0,0,0),
\]
and, thus,
\[
  (2,2,0)=(2,0,0)+2(0,1,0).
\]
Similarly, to show $(3,0,1)$ is in the span of~$T$, we
set~$(c_{3},c_{5})=(0,1)$.  The corresponding solution is
\[
    \left(\begin{array}{c}
	c_{1}\\
	c_{2}\\
	c_{3}\\
	c_{4}\\
	c_{5}
    \end{array}\right)
    =
    \left(\begin{array}{r}
	-\frac{3}{2}\\
	3\\
	0\\
	-1\\
	1
    \end{array}\right)
  \]
Therefore,
\[
  -\frac{3}{2}(2,0,0)+3(0,1,0)+0\cdot(2,2,0)-1\cdot(0,3,1)+1\cdot(3,0,1)=(0,0,0),
\]
Solving for~$(3,0,1)$ gives
\[
  (3,0,1)=\frac{3}{2}(2,0,0)-3(0,1,0)+(0,3,1).
\]
We summarize the underlying important algorithm: Let~$S=\{v_1,\ldots,v_k\}\in F^n$.  
To find a linearly independent subset~$T$ of~$S$ such that~$\Span(T)=\Span(S)$:
  \begin{itemize}[itemsep=1pt]
    \item Let~$M$ be the matrix with \emph{columns} $v_1,\ldots,v_k$.
    \item Compute~$M'$, the row-reduced form of~$M$.
    \item Let~$j_1,\ldots,j_d$ be the indices of the pivot columns of~$M'$ (the
      ones containing the leading $1$s).
    \item Set $T = \left\{v_{j_1},\ldots,v_{j_d}\right\}$.
  \end{itemize}
  \medskip

  \textbf{Note:} The set~$T$ is a subset of the columns of~$M$ \emph{not}
  of~$M'$!

  The elements of $S\setminus T$ correspond to the free variables, and we can
  write these elements as linear combinations of the elements of~$T$
  by setting each free variable in turn equal to~$1$ and setting the remaining free
  variables equal to~$0$.
\bigskip

We end with a result of fundamental importance:
\medskip

\textbf{Theorem.} Let~$S\subseteq V$ be linearly independent, and
let~$v\in\Span(S)$.  Then~$v$ has a unique expression as a linear
combination of elements of~$S$.  In other words, if $v=\sum_{i=1}^ka_iu_i$ and 
$v=\sum_{i=1}^{\ell}b_iw_i$ for some nonzero~$a_i,b_i\in F$ and
some~ distinct $u_i\in S$ and distinct~$w_i\in S$, then up to re-indexing, we have~$k=\ell$,~$u_i=w_i$,
and~$a_i=b_i$ for all~$i$.
\smallskip

\begin{proof}
  Say~$v=\sum_{i=1}^{n}a_iu_i$ and~$v=\sum_{i=1}^{n}b_iu_i$ for
  some $a_i,b_i\in F$ and~$u_i\in S$.  (By letting some~$a_i$ and~$b_i$ equal
    zero, these expressions represent two arbitrary representations of~$v$ as
    linear combinations of elements of~$S$, i.e., we can use the same~$u_i$
  and~$n$ for both expressions.)  It follows that
  \[
    0=v-v=\sum_{i=1}^{n}a_iu_i-\sum_{i=1}^{n}b_iu_i=\sum_{i=1}^{n}(a_i-b_i)u_i.
  \]
  Since~$S$ is linearly independent, it follows that~$a_i-b_i=0$ for
  all~$i$.  The result follows.
\end{proof}

\textbf{Example.} The previous result does not hold if~$S$ is linearly dependent.
For instance, consider the set~$S=\left\{ (1,1),(2,2) \right\}\subset\R$.  Then
\[
  (3,3)=(1,1)+(2,2)=2(1,1)+\frac{1}{2}(2,2)=3(1,1)+0(2,2)=\text{ etc.}
\]
\medskip

\textbf{Exercise.} Prove that the converse of the previous proposition holds: if
each element of~$\Span{S}$ can be expressed uniquely as a linear combination of
elements of~$S$, then~$S$ is linearly independent. 
\end{document}
