% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.

% Make two column format for LaTex 2e.\
\documentclass[11pt,twocolumn]{article} %,twocolumn

%\usepackage{times,amsmath,amsfonts}

% Use following instead for LaTex 2.09 (may need some other mods as well).
%\documentstyle[times,twocolumn]{article}
\usepackage[dvips]{graphicx,graphics}
% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{10in} \setlength{\textwidth}{7in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.5in}  % Centers text.
\setlength{\evensidemargin}{-.5in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-8pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-3pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand {\diff}[1] {\frac{\partial}{\partial #1}}
\newcommand{\jacob}[3]{\frac{\partial^2 #3}{\partial #1 \partial #2}}
\newcommand{\der}[2]{\frac{\partial #2}{\partial #1}}
\begin{document}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 471: Lecture 7\\
Generalized Inverse.}
% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
% For two authors (default example)
\maketitle \thispagestyle{empty}

\begin{enumerate}


\item $$\left(%
\begin{array}{cc}
  2 & 1 \\
  4 & 2 \\
\end{array}%
\right)\left(%
\begin{array}{c}
  y_1 \\
  y_2 \\
\end{array}%
\right) = \left(%
\begin{array}{c}
  1 \\
  1 \\
\end{array}%
\right).$$ There is no exact solution but it is possible to get a
solution in the least-squares sense. Note that
$$2y_1 + y_2 -1 = \epsilon_1, 4y_1 + y_2 -1 = \epsilon_2.$$
Minimize the sum of squared errors, $\min (X\beta -c)'(X\beta-c),$
where $c=(1,1)', \beta=(y_1,y_2)'$ and $X=\left(%
\begin{array}{cc}
  2 & 1 \\
  4 & 2 \\
\end{array}%
\right).$ $\frac{\partial}{\partial \beta} \beta'A = A,
\frac{\partial}{\partial \beta} \beta'A\beta = 2A\beta.$ Then it
can be shown that $\hat \beta = (X'X)^{-1}X'c$. The generalized
inverse (psudo inverse) of $X$ is $$X^{-}=(X'X)^{-1}X'.$$

\item Moore-Penrose generalized inverse $X^{-}$ is a unique matrix
that satisfies the following matrix equations
$$XX^{-}X=X, \; X^{-}XX^{-} = X^{-}$$
$$(XX^{-})'=XX^{-}, \; (X^{-}X)'=X^{-}X.$$
(see C.R. Rao and S.K. Mitra, Generalized inverse of matrices and
its applications, Wiley, New Work, 1971; Penrose, R., A
Generalized Inverse for Matrices. Proc. Cambridge Phil. Soc. 51,
406-413, 1955.)
\begin{verbatim}
>>X=[2 1
     4 2]
>>Y=pinv(X)
Y =
    0.0800    0.1600
    0.0400    0.0800
>> X*Y*X
ans =
    2.0000    1.0000
    4.0000    2.0000
\end{verbatim}

\item Computing the generalized inverse is based on the singular
value decomposition (SVD). Let $X$ be $n \times p$ matrix with $n
\geq p$. Then SVD of X is
$$X=UDV',$$
where $U_{n \times p}$ has orthonormal columns, $D_{p \times
p}=Diag(d_1,\cdots,d_p)$ is diagonal with non-negative elements
and $V_{p \times p}$ is orthogonal. For symmetric positive
definite matrix $X_{p \times p}$, we have diagonalization $X=QDQ'$
(lecture 07). For nonsingular matrix $X_{n \times n}$, $X=UDV'$
and $U$ is orthogonal. Let $D^{-}=Diag(d_1^{-},\cdots,d_p^{-}),$
$d_i^{-} = 1/d_i$ if $d_i \neq 0$ and $d_i^{-} = 0$ if $d_i=0$.
Then it can be shown that the Moore-Penrose generalized inverse is
given by
$$X^{-}=VD^{-}U'.$$

\begin{verbatim}
>> [U,D,V]=svd(X)
D =
    5.0000         0
         0    0.0000
>> U*D*V'
ans =
    2.0000    1.0000
    4.0000    2.0000
>> V*[1/5 0
       0  0]*U'
ans =
    0.0800    0.1600
    0.0400    0.0800
\end{verbatim}
\end{enumerate}
\end{document}