% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.
% Make two column format for LaTex 2e.\
\documentclass[11pt,twocolumn]{article} %,twocolumn
\usepackage{times,amsmath,amsfonts}

% Use following instead for LaTex 2.09 (may need some other mods as well).
%\documentstyle[times,twocolumn]{article}
\usepackage[dvips]{graphicx,graphics}
% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{10.5in} \setlength{\textwidth}{7.6in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-0.6in}  % Centers text.
\setlength{\evensidemargin}{-0.5in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-5pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-2pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray*}}
\newcommand{\eqn}{\end{eqnarray*}}
\newcommand{\bq}{\begin{eqnarray}}
\newcommand{\eq}{\end{eqnarray}}

\begin{document}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 471: Lecture 30\\
Cross-Validation.}
% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
% For two authors (default example)
\maketitle \thispagestyle{empty}
\begin{enumerate}
\item Given models $$M_{\sigma}: Y = g(x, \sigma^2) + \epsilon,$$
we wish to choose a best model. Suppose we estimate $g$ by
nonparametric regression such as kernel smoothing with bandwidth
$\sigma^2$: $$\hat g(x_i,\sigma^2) = \sum_{i=1}^m w_iy_i$$ where
$$ w(y_i,y) = \frac{\exp \big(-\frac{\|y_i - y\|^2}{2\sigma^2} \big)}
{\sum_{i=0}^m \exp \big(-\frac{\|y_i - y\|^2}{2\sigma^2}\big)}.$$


Among all possible models, we choose a model that give the
smallest possible prediction error. The residual is $$e_i = y_i -
\hat y_i = y_i - \widehat g(x_i,\sigma^2).$$ The residual sum of
squares (RSS) of a model $\sum_{i=1}^n e_i^2$ gives a measure of
predictive ability. However RSS is not a good model selector. Most
complex model that overfits data will have the smallest RSS. It is
due to the fact that $\hat y_i$ uses $y_i$ as well as all other
observations.
\begin{verbatim}
x=-30:30;
g=0.2*x.*exp(-x.^2/100) + cos(x/30).^2;
y=g+0.2*randn(1,61);
kernel=inline('exp(-x.^2/sigma^2)/
sum(exp(-x.^2/sigma^2))','x','sigma')
weight=kernel([-2:2],2);
yhat = convn(y,weight,'same');
>>sum((y-yhat).^2)
ans = 0.6445 %sigma=1
ans = 1.3638 %sigma=2
\end{verbatim}

\item To fix the above problem, we leave out $y_i$ when we predict
$y_i$ and call the prediction $\hat y_{(i)}$ and let
$$e_{(i)} = y_i - \hat y_{(i)}.$$
Then the predicted residual errors (PRESS) is $\sum_{i=1}^n
e_{(i)}$. It is also called the {\em cross-validation} (CV)
statistic. The cross-validation splits data into two disjoint sets
to fit a model in one set and to validate the model in the other
set. PRESS uses {\em leave-one-out} cross-validation. In kernel
smoothing, the prediction depends on parameter $\sigma^2$, so we
choose $\sigma^2$ such that we minimize
$$CV(\sigma^2) = \sum_{i=1}^n (y_i - \widehat g_{-i}(x_i,\sigma^2))^2.$$
where $\widehat g_{-i}(x_i,\sigma^2)$ is the kernel smoothing
estimator evaluated at $x_i$ when $(x_i,y_i)$ are removed.
\begin{verbatim}
for j=1:50
  sigma=j/20;
  for i=3:59 %forget boundary data
    weight=kernel([-2 -1 1 2],sigma);
    yi=[y(i-2) y(i-1) y(i+1) y(i+2)]';
    yhat(i)=weight*yi;
  end
  cv(j)=sum((y(3:59)-yhat(3:59)).^2);
end;
plot(cv)
>> find(cv==min(cv))
ans =
    50 %sigma=2.5
\end{verbatim}
\end{enumerate}
\begin{figure}[t]
\centering
\renewcommand{\baselinestretch}{1}
\includegraphics[scale=0.27]{lecture30-1.eps}
\includegraphics[scale=0.27]{lecture30-2.eps}
\includegraphics[scale=0.27]{lecture30-3.eps}
%\includegraphics[scale=0.32]{lecture28-5.eps}
\end{figure}

\end{document}