\documentclass[11pt]{article}

\usepackage{classDM}
\usepackage{hyperref}
\usepackage[normalem]{ulem}

\title{Asmt 7: Dimensionality Reduction}
\author{Turn in through Canvas by 2:45pm: \\
Monday, April 3 \\
100 points}
\date{}



\begin{document}
\maketitle

%\end{titlepage}




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Overview}

In this assignment you will explore regression techniques on high-dimensional data.  

You will use a data set for this assignment:
\begin{itemize} \denselist
\item \href{http://www.cs.utah.edu/~jeffp/teaching/cs5140/A7/A.dat}{\texttt{http://www.cs.utah.edu/\~{}jeffp/teaching/cs5140/A7/A.dat}}
\end{itemize}
and a file stub:
\begin{itemize} \denselist
\item \href{http://www.cs.utah.edu/~jeffp/teaching/cs5140/A7/FD.m}{\texttt{http://www.cs.utah.edu/\~{}jeffp/teaching/cs5140/A7/FD.m}}
\end{itemize}

These data sets are in matrix format and can be loaded into MATLAB or OCTAVE.  By calling 
\\
\texttt{load filename} (for instance \texttt{load X.dat})
\\
it will put in memory the data in the file, for instance in the above example the matrix \texttt{X}.  You can then display this matrix by typing 

\texttt{X}

\vspace{.1in}

\emph{As usual, it is highly recommended that you use LaTeX for this assignment.  If you do not, you may lose points if your assignment is difficult to read or hard to follow.  Find a sample form in this directory:
\url{http://www.cs.utah.edu/~jeffp/teaching/latex/}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Singular Value Decomposition (50 points)}

First we will compute the SVD of the matrix \texttt{A} we have loaded

\noindent
\texttt{[U,S,V] = svd(A)}

Then take the top $k$ components of \texttt{A} for values of $k = 1$ through $k=10$ using

\noindent
\texttt{Uk = U(:,1:k)}
\\ \noindent
\texttt{Sk = S(1:k,1:k)}
\\ \noindent
\texttt{Vk = V(:,1:k)}
\\ \noindent
\texttt{Ak = Uk*Sk*Vk'}

\paragraph{A (30 points):}
Compute and report the $L_2$ norm of the difference between \texttt{A} and \texttt{Ak} for each value of $k$ using

\noindent
\texttt{norm(A-Ak,2)}

\paragraph{B (10 points):}
Find the smallest value $k$ so that the $L_2$ norm of \texttt{A-Ak} is less than 10\% that of \texttt{A}; $k$ might or might not be larger than $10$.  

\paragraph{C (10 points):}
Treat the matrix as 5000 points in 40 dimensions.  Plot the points in $2$ dimensions in the way that minimizes the sum of residuals squared, and describe briefly how you did it.   




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Frequent Directions and Random Projections (50 points)}

Use the stub file $\texttt{FD.m}$ to create a function for the Frequent Directions algorithm (\textbf{Algorithm 16.2.1}).  We will consider running this code on matrix \texttt{A}.  

\paragraph{A (25 points):}
We can measure the error $\max_{\|x\|=1} | \|A x\|^2 - \|B x\|^2 |$ as 
\texttt{norm(A'*A - B'*B, 2)}.  
\begin{itemize} \denselist
\item How large does \texttt{l} need to be for the above error to be at most $\|A\|_F^2 /10$?  
\item How does this compare to the theoretical bound (e.g. for $k=0$).  
\item How large does \texttt{l} need to be for the above error to be at most $\|A - A_k\|_F^2/10$ (for $k=2$)?
\end{itemize}

Note: you can calculate $\|A\|_F^2$ as \texttt{norm(A, 'fro')\^{}2}.  


\paragraph{B (25 points):}
Create another \texttt{l x d} matrix $B$, but using random projections.  You can do this by creating an \texttt{l x n} matrix \texttt{S}, and letting \texttt{B = SA}.  Fill each entry of \texttt{S} by an independent normal random variable $S_{i,j} = $ {\color{red}$\frac{\sqrt{\texttt{l}}}{\sqrt{n}} N(0,1)$} 
(\textbf{not {\color{yellow}
$\frac{\sqrt{n}}{\sqrt{\texttt{l}}} N(0,1)$
}}).  

Estimate how large should \texttt{l} be in order to achieve $\max_{\|x\|=1} | \|A x\|^2 - \|B x\|^2 | \leq \|A\|_F^2/10$.  To estimate the relationship between \texttt{l} and the error in this randomized algorithm, you will need to run multiple trials.  Be sure to describe how you used these multiple trials, and discuss how many you ran and why you thought this was enough trials to run to get a good estimate.  




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{BONUS (2 points)}

Consider any $n \times d$ matrix $A$.  For some parameter $t \geq 1$, assume that $n > d > 100 t^2$.  Without knowing $A$, explain at most how many squared singular values of $A$ could be larger than $\|A\|_F^2/ t$.  


\end{document}
