\documentclass[11pt,twocolumn]{article}
\usepackage{times,amsmath,amsfonts}
\usepackage[dvips]{graphicx,graphics}



\setlength{\textheight}{10in} \setlength{\textwidth}{7.1in}
\setlength{\columnsep}{0.3in} \setlength{\topmargin}{0in}
\setlength{\headheight}{-0.3in} \setlength{\headsep}{0in}
\setlength{\oddsidemargin}{-0.6in}  % Centers text.


% Add the period after section numbers.  Adjust spacing.
%\newcommand{\Section}[1]{\vspace{-5pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
%\newcommand{\SubSection}[1]{\vspace{-2pt}\subsection{\hskip -1em.~~#1}
%        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray*}}
\newcommand{\eqn}{\end{eqnarray*}}
\newcommand{\bq}{\begin{eqnarray}}
\newcommand{\eq}{\end{eqnarray}}

\begin{document}

\title{Stat 992: Lecture 10\\
Hilbert Space.}
\author{Moo K. Chung \tt{mchung@stat.wisc.edu}}
\maketitle \thispagestyle{empty}

%\begin{figure}
%\centering
%\renewcommand{\baselinestretch}{1}
%\includegraphics[scale=0.8]{homework05-1.eps}
%\end{figure}

\begin{enumerate}

\item {\em Hilbert space.} In the previous lecture, we showed how
to generate Gaussian fields using simple kernel smoothing. A more
general technique is based on orthogonal expansions of the fields.

A Hilbert space $\mathcal{H} $ is a complete inner product space.
See F. Riesz and B. Sz.-Nagy's Functional Analysis (1955). For
every $f,h \in \mathcal{H}$, there is a scalar number $\langle f,h
\rangle$ such that mapping $f \to \langle f, h \rangle$ is a
linear functional, i.e. $\langle \alpha f + \beta g, h \rangle =
\alpha \langle f,h \rangle + \beta \langle g, h \rangle$. Letting
$\alpha=0$ and $\beta =0$, $\langle 0, h \rangle = 0$. Further we
assume $\langle f, g \rangle = \langle g, f \rangle$ and $\langle
f, f \rangle > 0$ if $f \neq 0$. We call such mapping $\langle
\cdot, \cdot \rangle$ an inner product. We define metric norm of
an element $f$ as $\| f\| = \langle f, f \rangle^{1/2}$.
$\mathcal{H}$ is complete if a sequence of elements $f_n \in
\mathcal{H}$ with the condition $\|f_n - f_m\| \to 0$ for $m,n \to
\infty$ has a limit in $\mathcal{H}$.

A space of functions $L^2(0,1)$ with inner product $$\langle f, g
\rangle = \int_0^1 f(x)g(x) \;dx$$ such that $\| f \| < \infty$ is
a Hilbert space. This space is usually denoted by $L^2(0,1)$. The
completeness of $L^2(0,1)$ space is a standard exercise in real
analysis.

\item {\em Orthogonal expansion.} If $U \subset \mathcal{H}$, the
span of $U$ denoted by $Span(U)$ is the collection of all finite
linear combinations of elements in $U$. Further define a closure
of $Span(U)$ to be the smallest closed set containing $Span(U)$.
For example consider $H=L^2(0,1)$. $U=\{1,x,x^2,\cdots\} \subset
H$, $Span(U)$ is the collection of all polynomials while
$\overline{Span}(U) =H$. So in this particular example, $U$ is
dense in $H$. The whole idea of orthogonal expansion is to find a
relatively smaller countable set $U$ that spans $H$. We call a set
separable if it contains a countable spanning subset. So
$L^2(0,1)$ is separable.

Most Hilbert spaces in practice are separable. So for separable
Hilbert space $\mathcal{H}$, we have a spanning set
$\{u_j\}_0^{\infty}$. Let $U= \{u_j\}_0^{m}$ be the subset of
independent elements. Then using the Gram-Schmidt
orthogonalization, we can construct orthonormal basis
$V=\{v_j\}_0^{m}$ from $U = \{u_j\}_0^{m}$ such that $$v_j =
\frac{w_j}{\|w_j\|}, w_j = u_j - \sum_{i=0}^{j-1} \langle u_j,e_i
\rangle e_i.$$ Then for any element $f \in \mathcal{H}$, $$f =
\sum_{j=0}^{\infty} \alpha_j v_j.$$ The coefficient of the
expansion is given by $\alpha_j = \langle f, v_j \rangle$ and it
is usually termed generalized fourier coefficients for the obvious
reason.

We can do a better approximation than this. Based on finite basis
$\{v_j\}_0^m$, we can approximate any element $f \in \mathcal{H}$
in the least-square sense by minimizing
$$\min_{\alpha_j} \Big|\Big|f - \sum_0^m \alpha_j v_j\Big|\Big|.$$
The minimum always exists in Hilbert space and this is one reason
why the least-squares estimation is a widely method. What happen
when we use additional basis $v_{m+1}$ to improve the
approximation? Nice thing about orthogonal expansion is that there
is no need to estimate $\alpha_j$ again since they are given by $
\alpha_j = \langle f, v_j \rangle$ for all $j=0,\cdots,m+1$.

{\bf Problem 20.} Construct orthonormal basis $v_j(x), x \in S^1$
that can be used to estimate an integrable function on a unit
circle. This can be used in representing functional data on a unit
circle, smoothing data via a heat equation etc. After
construction, simulate Gaussian random field $\sum_{j=0}^m
Z_jv_j(x)$ where i.i.d. $Z_j \sim N(0,\sigma^2)$ in computer. What
happens when you add additional basis $v_{j+1}$?

\end{enumerate}

\end{document}