\documentclass[11pt,twocolumn]{article}
\usepackage{times,amsmath,amsfonts}
\usepackage[dvips]{graphicx,graphics}



\setlength{\textheight}{10in} \setlength{\textwidth}{7.1in}
\setlength{\columnsep}{0.3in} \setlength{\topmargin}{0in}
\setlength{\headheight}{-0.3in} \setlength{\headsep}{0in}
\setlength{\oddsidemargin}{-0.6in}  % Centers text.


% Add the period after section numbers.  Adjust spacing.
%\newcommand{\Section}[1]{\vspace{-5pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
%\newcommand{\SubSection}[1]{\vspace{-2pt}\subsection{\hskip -1em.~~#1}
%        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray*}}
\newcommand{\eqn}{\end{eqnarray*}}
\newcommand{\bq}{\begin{eqnarray}}
\newcommand{\eq}{\end{eqnarray}}

\begin{document}

\title{Stat 992: Lecture 11\\
Karhunen-Lo$\grave{e}$ve Expansion.}
\author{Moo K. Chung \tt{mchung@stat.wisc.edu}}
\maketitle \thispagestyle{empty}

%\begin{figure}
%\centering
%\renewcommand{\baselinestretch}{1}
%\includegraphics[scale=0.8]{homework05-1.eps}
%\end{figure}

\begin{enumerate}


\item {\em Orthogonal expansion for fields} Let $\mathcal{G} =
L^2(G(\Omega))$ be the space of zero mean Gaussian random fields
in $\Omega \subset \mathbb{R}^n$ with inner product
$$\langle X, Y \rangle = \mathbb{E} \int_{\Omega} X(x)Y(x) \;dx$$
with $\|X\| < \infty$. This is basically the integral of
cross-covariance function between fields $X$ and $Y$.
$\mathcal{G}$ can be shown to be a separable Hilbert space. This
can be shown by finding countable orthonormal basis in
$\mathcal{G}$. See Random Fields and Their Geometry by R.J. Adler
and J.E. Taylor.

Let $\psi_i$ be basis for $L^2(\Omega)$ and $Z_i$ be i.i.d.
$N(0,\sigma_i^2)$. We will construct orthonormal expansion in the
following fashion. Suppose for any $X \in \mathcal{G}$,
$$X(x) = \sum_{i=0}^{\infty} Z_i \psi_i(x).$$

Note that $\mathbb{E} X(x) = 0$ while the covariance function is
\bqn R(x,y) &=& \mathbb{E} \sum_{i,j=0}^{\infty} Z_i
\psi_i(x)Z_j\psi_i(y)\\
&=&\sum_{i=0}^{\infty} \sigma_i^2 \psi_i(x)\psi_i(y) . \eqn The
covariance function of zero mean Gaussian field completely
characterizes the field itself so we need to know if any
covariance function can be expressed this way. Obviously $R$ has
to be symmetric to be expressible this way.

\item {\em Degenerate kernels}. We follow the argument of R.
Courant and D. Hilbert (Methods of Mathematical Physics, Vol I.
1953). Suppose we fix $y$. Then from the Weierstrass's
approximation theorem, for continuous function, $R(x,\cdot) =
\sum_{i=0}^{\infty} \alpha_i(x)$ for some basis functions
$\alpha_i$ uniformly. Now fix $x$ and we have \bqn R(x,y) &=&
\sum_{i=0}^{\infty} \alpha_i(x) \sum_{j=0}^{\infty}
\beta_j(x)\\
&=& \sum_{i,j=0}^\infty \alpha_i(x)\beta_j(x) \eqn Again,
$\beta_j$ are basis functions. Courant and Hilbert termed such the
kernel degenerate. For finite number of basis
$\alpha_1,\cdots,\alpha_p$ and $\beta_1,\cdots,\beta_p$, it can be
representable as a linear combination of orthogonal basis $\psi$
using the Gram-Schmidt orthogonalization. So we have
$$R(x,y) = \sum_{i,j=0}^{\infty} c_{ij}\psi_i(x)\psi_j(x).$$
Assume further that the kernel is symmetric then
$$\sum_{i,j=0}^{p} (c_{ij}-c_{ji})\psi_i(x)\psi_j(x)=0$$
for $p \to \infty$. Since $\psi_i$ and $\psi_j$ are independent,
$c_{ij} = c_{ji}$. We can do a better job than this via Mercer's
theorem (Read Riesz and Sz.-Nagy, 1956; Courant and Hilbert. 1953;
Adler and Taylor, 2004).

\item {\em Mercer's theorem.} For continuous symmetric kernel $R$,
define linear operator $\mathcal{R}: L^2(\Omega) \to L^2(\Omega)$
as
$$\mathcal{R}f(x) = \int_{\Omega} R(x,y) f(y) \;dy.$$
This is a compact self-adjoint operator\footnote{$R$ is compact if
the closure of $R( \mbox{close ball} )$ is compact and it is
self-adjoint if $\langle \mathcal{R}f,g \rangle = \langle
f,\mathcal{R}g \rangle$. See J.B. Conway's A Course in Functional
Analysis, 2nd edition (1990) for the discussion of compact
operators on Hilbert space. S.C. Joshi's PhD thesis (1998) Large
Deformation Diffeomorphisms and Gaussian Random Fields for
Statistical Characterization of Brain Sub-manifolds for it's use
in actual image analysis.} which yields unique countable
eigenvalues $\lambda_i$ and orthonormal eigenfunctions $\phi_i$
such that $\mathcal{R}\phi_i = \lambda_i \phi_i$\footnote{This
eigenequation is called a Fredholm equation of the first kind.
Given covariance function, there is a numerical technique for
estimating $\lambda_i$ and $\phi_i$ numerically.} with
$\lambda_{\infty} =0$. We will order them such that $\lambda_0 >
\lambda_1 > \lambda_2 > \cdots$.

The compact self-adjoint operator $\mathcal{R}$ has a spectral
representation $\mathcal{R}f = \sum_{i=0}^{\infty} \lambda_i
\langle \phi_i, f \rangle \phi_i.$\footnote{See Conway Theorem
7.6.} Mercer's theorem stats that
$$R(s,t) = \sum_{i=0}^{\infty} \lambda_i \phi_i(x)\phi_i(y).$$
Hence identifying $\lambda_i=\sigma_i^2$ and $\phi_i =\psi_i$, we
have the desired orthonormal expansion and it is called the
Karhunen-Lo$\grave{e}$ve expansion.

To prove the theorem, note that $\phi_i(x)\phi_j(y)$ forms basis
in $\Omega \otimes \Omega$. This is how you expand basis function
in 1 dimension to higher dimension in Euclidean space. Assuming
$R(x,y) \in L^2(\Omega \otimes \Omega)$,
$$R(x,y) = \sum_{i,j=0}^{\infty} c_{ij}\phi_i(x)\phi_j(y).$$
Then \bqn \int_{\Omega}R(x,y)\phi_k(y) \;dy &=&
\sum_{i,j=0}^{\infty} c_{ij}\int_{\Omega}\phi_k(y)\phi_i(x)\phi_j(y)\;dy\\
\lambda_k \phi_k(x)&=&\sum_{i,j=0}^{\infty} c_{ij}
\phi_i(x)\delta_{jk} = \sum_{i=0}^{\infty}d_i\phi_i(x)\eqn where
$d_i = \sum_{k=0}^{\infty} c_{ik}.$ Since they are linearly
independent, $c_j =\lambda_j$.

{\bf Problem 21.} Let $X$ be a Brownian motion with covariance
function given by $R(x,y) = \min(x,y)$. Find the eigenvalues and
eigenfunctions. Based on this eigensystem, expand the Brownian
motion via the Karhunen-Lo$\grave{e}$ve expansion. Based on the
Karhunen-Lo$\grave{e}$ve expansion, simulate Brownian motion in
$\tt{MATLAB}$.

{\bf Problem 22.} Without using the property of compact
self-adjoint operator in Hilbert space, prove Mercer's theorem.
Follow Riesz and Sz.-Nagy's classical derivation.

\end{enumerate}

\end{document}