% $Header: /cvsroot/latex-beamer/latex-beamer/examples/beamerexample5.tex,v 1.22 2004/10/08 14:02:33 tantau Exp $

\documentclass[11pt]{beamer}
\usepackage{amscd}

%\usetheme{Madrid} % Darmstadt, Boadilla


\mode<presentation> {
  \usetheme{Madrid}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsbsy,alltt,amsmath,times}
\usepackage{epsfig, color, graphicx, rotate, colordvi}
%\usepackage{fancybox,fancyhdr,times,subfigure}
\usepackage{hyperref, mathptmx}


\setbeamercovered{dynamic}

\newcommand{\Lang}[1]{\operatorname{\text{\textsc{#1}}}}

\newcommand{\Class}[1]{\operatorname{\mathchoice
  {\text{\sf \small #1}}
  {\text{\sf \small #1}}
  {\text{\sf #1}}
  {\text{\sf #1}}}}

\newcommand{\NumSAT}      {\text{\small\#SAT}}
\newcommand{\NumA}        {\#_{\!A}}

\newcommand{\barA}        {\,\bar{\!A}}

\newcommand{\Nat}{\mathbb{N}}
\newcommand{\Set}[1]{\{#1\}}

\newtheorem{thm}{Result}%[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{defn}[thm]{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{rem}{Remark}

\newcommand{\cred}{\textcolor {red}}
\newcommand{\cblue}{\textcolor {blue}}
\newcommand{\cgreen}{\textcolor {green}}

\def\ul{\underline}

\def\sign{\;{\rm sign}}
\def\rank{\;{\rm rank}}
\def\disp{\displaystyle}
\def\sn{\sqrt{n}}
\def\Pr{\mathrm{P}}

\def\Cov{\;{\rm Cov}}
\def\L{\underline L}
\def\LL{\overline L}

\def\balpha{{\boldsymbol{\alpha}}}
\def\cal{\mathcal}
\def\conas{\stackrel{a.s.} {\longrightarrow}}         % conv. a.s.
\def\conp{\stackrel{\cal P} {\longrightarrow}}        % conv. in probability
\def\conD{\stackrel{\mathcal{L}} {\longrightarrow}}        % conv. in distribution
\def\conw{\stackrel{\mathcal{L}} {\rightarrow}}        % conv. in distribution
\def\hat{\widehat}
%\def\endproof{\begin{flushright}$\lozenge$\end{flushright}}
\def\Proof{\noindent{\it Proof }}
\def\endproof{\hfill{$\Box$}}
\newcommand{\wcon}{\stackrel{\mathcal{L}}{\longrightarrow}}
\def\Var{\mbox{var}}
\def\br{\begin{array}}
\def\er{\end{array}}
\def\ep{\epsilon}
%%%%%%%%%%%%%%% MY DEFINITIONS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Put your own definitions
%
\def\VaR{{\rm VaR}}
\def\E{{\rm E}}
\def\beqrs{\begin{eqnarray*}}           %\beqrs
\def\eeqrs{\end{eqnarray*}}             %\eeqrs

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\pgfdeclaremask{tu}{beamer-tu-logo-mask}
\pgfdeclaremask{computer}{beamer-computer-mask}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerimage}{beamer-computer}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerworkingimage}{beamer-computerred}
\pgfdeclareimage[mask=tu,height=.5cm]{logo}{beamer-tu-logo}

\logo{\pgfuseimage{logo}}

\title[Stat 609 Lecture 19]{Stat 609: Mathematical Statistics \\ Lecture 19}
\author[UW-Madison]{Jun Shao}
\institute[Statistics]{%
%  Department of Statistics\\
%  University of Wisconsin\\
%  Madison, WI 53706, USA\\
%~~~\\
  }
\date[2015]{}

\colorlet{redshaded}{red!25!bg}
\colorlet{shaded}{black!25!bg}
\colorlet{shadedshaded}{black!10!bg}
\colorlet{blackshaded}{black!40!bg}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\def\radius{0.96cm}
\def\innerradius{0.85cm}

\def\softness{0.4}
\definecolor{softred}{rgb}{1,\softness,\softness}
\definecolor{softgreen}{rgb}{\softness,1,\softness}
\definecolor{softblue}{rgb}{\softness,\softness,1}

\definecolor{softrg}{rgb}{1,1,\softness}
\definecolor{softrb}{rgb}{1,\softness,1}
\definecolor{softgb}{rgb}{\softness,1,1}

\newcommand{\Bandshaded}[2]{
  \color{shadedshaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\Band}[2]{
  \color{shaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\BaenderNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{input tapes}{}}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{Zwei Eingabebänder}{}}
  \pgfputat{\pgfxy(0,4.25)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \ignorespaces}

\newcommand{\BaenderHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{input tapes}{}}
  \color{shaded}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Bandshaded{}{}}

  \color{blackshaded}
  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{Zwei Eingabebänder}{}}%
  \color{blackshaded}
  \pgfputat{\pgfxy(0,4.25)}{\Bandshaded{}{}}
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}%

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \ignorespaces}

\newcommand{\Slot}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.0)}%
    \pgfbezier{\pgfxy(0.1,3.95)}{\pgfxy(0.05,3.9)}{\pgfxy(0,3.9)}%
    \pgfbezier{\pgfxy(-0.05,3.9)}{\pgfxy(-0.1,3.95)}{\pgfxy(-0.1,4.0)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\SlotZwei}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.25)}%
    \pgfbezier{\pgfxy(0.1,4.25)}{\pgfxy(0.05,4.15)}{\pgfxy(0,4.15)}%
    \pgfbezier{\pgfxy(-0.05,4.15)}{\pgfxy(-0.1,4.2)}{\pgfxy(-0.1,4.25)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\ClipSlot}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4cm}}}{\pgfxy(0.2,1.5)}\ignorespaces}

\newcommand{\ClipSlotZwei}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4.25cm}}}{\pgfxy(0.2,1.25)}\ignorespaces}


\AtBeginSection[]{\frame{\frametitle{Outline}\tableofcontents[current]}}

\begin{document}

%\frame{\titlepage}

\frame {\vspace{-2mm}
\frametitle{
Lecture 19: Convergence}
\begin{alertblock}{Asymptotic approach}
\begin{itemize}
  \item In statistical analysis or inference,
a key to the success
of finding a good procedure is being able to find some moments
and/or distributions  of various statistics.
  \item In many complicated problems we are not able to find exactly
the moments or distributions of given statistics.
  \item When the sample size $n$ is large,
we may approximate the moments and distributions of
statistics, using asymptotic tools, some of which are studied in this course.
  \item In an asymptotic analysis, we consider a sample $X=(X_1,...,X_n)$
not for fixed $n$, but as a member of a sequence corresponding to $n=
n_0,n_0+1,...$, and obtain the limit of the distribution of an
appropriately normalized statistic or variable
$T_n(X)$ as $n\rightarrow \infty$.
  \item The limiting distribution and its moments are used as approximations
to the distribution and moments of $T_n(X)$
in the situation with a large but actually finite $n$.
\end{itemize}
\end{alertblock}}
\frame{\vspace{-2mm}
    \begin{alertblock}{ }
\begin{itemize}
  \item This leads to some asymptotic statistical procedures and asymptotic
criteria for assessing their performances.
  \item The asymptotic approach is not only applied  to the situation where no
exact method  (the approach considering a fixed
$n$) is available, but also  used to provide a
 procedure simpler (e.g., in terms of computation)
than that produced by the exact approach.
\item In addition to providing more theoretical results and/or simpler procedures,
the asymptotic approach requires less stringent mathematical assumptions
than does the exact approach.
\end{itemize}
     \end{alertblock}\vspace{-2mm}
     \begin{block}{Definition 5.5.1 (convergence in probability)}
A sequence of random variables $Z_n$, $i=1,2,...$,
converges in probability to a random variable $Z$ iff for every $\ep >0$,\vspace{-2mm}
\[ \lim_{n\rightarrow \infty} P(|Z_n - Z| \geq \ep ) = 0 . \vspace{-2mm} \]
A sequence of random vectors $Z_n$
converges in probability to a random vector $Z$ iff each component of
  $Z_n$ converges in probability to the corresponding component of $Z$.
     \end{block}
}
\frame{\vspace{-2mm}
\begin{alertblock}{Theorem 5.5.2 (Weak Law of Large Numbers (WLLN))}
Let $X_1,...,X_n$ be iid random variables with $E(X_i)= \mu $ and finite Var$(X_i)= \sigma^2 $.
Then, the sample mean $\bar{X}$ converges in probability to $\mu$.
\end{alertblock}\vspace{-2mm}
\begin{alertblock}{Proof.}
 By Chebychev's inequality and Theorem 5.2.6,\vspace{-2mm}
\[  P(|\bar{X}-\mu|\geq \ep )  \leq  \frac{{\rm Var}(\bar{X})}{\ep^2}
 = \frac{\sigma^2}{n\ep^2} \vspace{-2mm}\]
which converges to 0 as $n\rightarrow \infty$.
\end{alertblock}\vspace{-2mm}
\begin{block}{Remarks.}
\begin{enumerate}
\item
Although we write the sample mean as $\bar{X}$, it depends on $n$.
\item
The WLLN states that the probability of the sample mean $\bar{X}$ being close to the population mean $\mu$
converges to 1.
\item
The existence of a finite variance $\sigma^2$ is not needed; we only need the existence of $E(X_i)$, a proof will be given later.
\item
The independence assumption is not necessary either: in the previous proof, we only need $X_i$'s are uncorrelated.
\end{enumerate}
\end{block}}
\frame{\vspace{-2mm}
\begin{alertblock}
{\bf Example.} Suppose that $X_1,...,X_n$ are identically distributed with $E(X_i)= \mu$ and Var$(X_i)=\sigma^2 < \infty$, and
that\vspace{-3mm}
\[ {\rm Cov}(X_t,X_s) = \left\{ \br{ll}
c & \quad \mbox{if } |s-t| =1 \vspace{2mm}\\
0 &  \quad \mbox{if } |s-t| >1\er \right. \vspace{-2mm} \]
Then $\bar{X}$ converges in probability to $\mu$, because  \vspace{-2mm}
\beqrs
 {\rm Var}(\bar{X}) &=& {\rm Var} \left( \frac{1}{n} \sum_{i=1}^n X_i\right)
= \frac{1}{n^2} {\rm Var} \left(  \sum_{i=1}^n X_i\right) \\
&=& \frac{1}{n^2} \left( \sum_{i=1}^n {\rm Var}(X_i) +
 \sum_{i \neq j} {\rm Cov}(X_i,X_j ) \right) \\
& = & \frac{\sigma^2}{n} + \frac{(n-1)c}{n^2}
\eeqrs
and, Chebychev's inequality, \vspace{-2mm}
\[ P(|\bar{X}-\mu|\geq \ep )  \leq \frac{{\rm Var}(\bar{X})}{\ep^2} = \frac{\sigma^2+ (1-n^{-1})c }{n\ep^2} \rightarrow 0\]
 \end{alertblock}}
 \frame{
 \begin{alertblock}{A proof of the WLLN using chf's}
 Let $X_1,...,X_n$ be iid random variables with $E|X_1|< \infty$ and $E(X_i) = \mu$.

From the result for the chf (Theorem C1), the chf of $X_1$ is differentiable at 0 and
\[ \phi_{X_1} (t) = 1 + \dot{\imath} \mu t + o(|t|)  \quad\quad \mbox{as $|t| \rightarrow 0$}. \]
Then, the chf of $\bar{X}$ is \vspace{-2mm}
\[ \phi_{\bar{X}} (t) = \left[\phi_{X_1} \! \left(\frac{t}{n}\right)\right]^n
= \left[ 1 + \frac{\dot{\imath} \mu t}{n} + o\left( \frac{t}{n}\right)\right]^n
\rightarrow e^{\dot{\imath} \mu t}  \vspace{-2mm}\]
for any $t \in {\cal R}$ as $n \rightarrow \infty$, because
$(1+c_n/n)^n \rightarrow e^c$ for any complex
sequence $\{ c_n \} $ satisfying $c_n \rightarrow c$.
\vspace{1mm}

The limiting function $e^{\dot{\imath} \mu t}$ is the chf of
the constant $\mu$.
\vspace{1mm}

By Theorem C7, if $F_{\bar{X}}(x)$ is the cdf of $\bar{X}$, then
\vspace{-2mm}
\[ \lim_{n\rightarrow \infty} F_{\bar{X}}(x) = \left\{ \br{ll}
1 & \quad x > \mu \\
0 & \quad x < \mu \er \right. \vspace{-2mm} \]
This shows
that $\bar{X}$ converges in probability to $\mu$, because of Theorem 5.5.13 to be established later.
 \end{alertblock}}
\frame{
\begin{block}{Theorem 5.5.4.}
Let $Z_1,Z_2,...$ be random vectors that converge in probability to a random vector $Z$ and let $h$ be a continuous function.
Then $h(Z_1),h(Z_2),...$ converges in probability to $h(Z)$.
\end{block}\vspace{-2mm}
\begin{alertblock}{Example 5.5.3.}
Let $X_1,X_2,...$ be iid random variable
with $E(X_i)=\mu$ and Var$(X_i)=\sigma^2$.

Consider the sample variance \vspace{-2mm}
\[ S^2 = \frac{1}{n-1}\sum_{i=1}^n (X_i-\bar{X})^2 = \frac{1}{n-1}\sum_{i=1}^n (X_i-\mu)^2 - \frac{n(\bar{X}-\mu)^2}{n-1} \vspace{-2mm}\]
Define \vspace{-2mm}
\[ Z_n = \frac{1}{n} \sum_{i=1}^n (X_i - \mu)^2 , \quad U_n = \bar{X} -\mu,
\quad a_n = \frac{n}{n-1} \vspace{-2mm} \]
By the WLLN,
$ (Z_n, U_n) $ converges in probability to $(\sigma^2 , 0)$.

Note that $a_n \rightarrow 1$ and $a_n$ is not random, but we can view that $a_n$ converges in probability to 1.

Then, by Theorem 5.5.4,
$S^2 = h(a_n, Z_n, U_n) = a_n ( Z_n -  U_n^2 )$ converges in probability to $h(1,\sigma^2, 0 ) = \sigma^2$.
\end{alertblock}}
\frame{\vspace{-2mm}
\begin{alertblock}{Example 5.5.5.}
Consider $h(x) = \sqrt{x}$.

By Theorem 5.5.4,  the sample standard deviation
$S=h(S^2)$ converges in probability to the population standard deviation $\sigma = h(\sigma^2)$.
\end{alertblock}
\vspace{-2mm}
\begin{block}{}
Note that convergence in probability is different from the convergence of a sequence of deterministic
functions $g_n(x)$ to a function $g(x)$ for $x$ in a set $A \subset {\cal R}^k$.

Similar to the convergence of deterministic functions (note that random variables are random functions),
we have the following concept.
\end{block}\vspace{-2mm}
\begin{block}{Definition 5.5.6 (convergence almost surely)}
A sequence of random variables $Z_n$, $n=1,2,...$,
converges almost surely to a random variable $Z$ iff \vspace{-2mm}
\[  P\left(\lim_{n\rightarrow \infty}Z_n =Z \right) = 1 . \vspace{-2mm} \]
A sequence of random vectors $Z_n$, $n=1,2,...$,
converges almost surely to a random vector $Z$
iff each component of $Z_n$ converges almost surely to the
corresponding component of $Z$.
\end{block}
}
\frame{
\begin{block}{}
\begin{itemize}
\item
The almost sure convergence of $Z_n$ to $Z$ means that there is an event $N$ such that $P(N)=0$ and for every
element $\omega \in N^c$, $\lim_{n\rightarrow \infty} Z_n(\omega ) = Z(\omega )$, which
is almost the same as point-wise convergence for deterministic functions (Example 5.5.7).
\item
If a sequence of random vectors $Z_n$ converges almost surely to a random vector $Z$, and $h$ is a continuous function,
then $h(Z_n)$ converges almost surely to $h(Z)$.
\item
If $Z_n$ converges almost surely to $Z$, then $Z_n$ converges in probability to $Z$.
\item
Convergence in probability, however, does not imply convergence almost surely (Example 5.5.8).
\item
If $Z_n$ converges in probability fast enough, then it converges almost surely, i.e.,
if for every $\ep >0$, \vspace{-2mm}
\[ \sum_{n=1}^\infty P(|Z_n - Z|\geq \ep ) < \infty, \vspace{-2mm}\]
then $Z_n$ converges almost surely to $Z$.
\end{itemize}
\end{block}}
\frame{\vspace{-2mm}
\begin{block}{}
It is, however, not easy to construct an example of convergence in probability but not almost surely.
\end{block}\vspace{-2mm}
\begin{alertblock}{}
Similar to the WLLN in Theorem 5.5.2, we have the following result with almost sure convergence.
\end{alertblock}\vspace{-2mm}
\begin{alertblock}{Theorem 5.5.9 (Strong Law of Large Numbers (SLLN))}
Let $X_1,...,X_n$ be iid random variables with $E(X_i)= \mu $.
Then, the sample mean $\bar{X}$ converges almost surely to $\mu$.
\end{alertblock}\vspace{-2mm}
\begin{alertblock}{}
Note that we still only require the existence of the mean, not the second order moment. 
\vspace{1mm}

The proof is omitted, since it is out of the scope of the textbook. 
 \end{alertblock}\vspace{-2mm}
         \begin{block}{Approximation to an integral}
         	Suppose that $h(x)$ is a function of $x \in {\cal R}^k$.
         	
         	In many applications we want to calculate an integral \vspace{-2mm}
         	\[ \int_{{\cal R}^k} h(x) dx  \]
       \end{block}}
                	    \frame{\begin{block}{}
                	    		If the integral is not easy to calculate, a numerical method is needed. 
                	    		\vspace{1mm}
                	    		
                	    		The following is the so called Monte Carlo approximation method, which is based on the SLLN. 
                	    		\vspace{1mm}
                	    		
                	    	
        	Suppose that we can generate iid random vectors $X_1,X_2,...$ from a pdf $p(x)$ on ${\cal R}^k$ satisfying
         	that $p(x)>0$ if $h(x) \neq 0$.\vspace{1mm}
         	
         	By the SLLN, 	with probability equal to 1 (almost surely),
         	\vspace{-2mm}
         	\beqrs
         	 \lim_{n\rightarrow \infty} \frac{1}{n} \sum_{i=1}^n \frac{h(X_i)}{p(X_i)}
         	& = & E\left( \frac{h(X_1)}{p(X_1)}\right) \\
         	& = & \int_{{\cal R}^k} \frac{h(x)}{p(x)} p(x) dx\\
         	 & = & \int_{{\cal R}^k} h(x) dx \vspace{-2mm} \eeqrs
         	    	Thus, we can approximate the integral by the average $ \frac{1}{n} \sum_{i=1}^n \frac{h(X_i)}{p(X_i)}$ with a very large $n$.
         	    	\vspace{1mm}
         	    	
         	    	We can actually find what is the large enough $n$ to have a good approximation.
         	    \end{block}}
 \frame {\vspace{-2mm}
 	\begin{alertblock}{}
 		We often need to consider a convergence even weaker than convergence in probability.
 	\end{alertblock}\vspace{-2mm}
 	\begin{block}{Definition 5.5.10 (convergence in distribution)}
 		A sequence of random variables $Z_n$, $n=1,2,...$, converges
 		in distribution to a random variable $Z$ iff \vspace{-2mm}
 		\[ \lim_{n\rightarrow \infty} F_{Z_n}(x) = F_Z(x) , \quad \quad x \in \{ y: F_Z(y)
 		\mbox{ is continous }\}\vspace{-2mm} \]
 		where $F_{Z_n}$ and $F_Z$ are the cdf's of $Z_n$ and $Z$, respectively.
 	\end{block}\vspace{-2mm}
 	\begin{block}{}
 		\begin{itemize}
 			\item
 			Note that we only need to consider the convergence at $x$ that is a continuity point of $F_Z$.
 			\item
 			Note that cdf's, not pdf's or pmf's, are involved in this definition.
 			\item
 			In convergence in distribution, it is really the cdfs that converge, not the random variables;
 			in fact, the random variables can be defined in different spaces, which
 			is very different from the convergence in probability or almost surely.
 		\end{itemize}
 	\end{block}}
 	\frame{ \vspace{-2mm}
 		\begin{alertblock}{Example 5.5.11.}
 			Let $X_1,X_2,...$ be iid from uniform on $(0,1)$ and
 			$X_{(n)} = \max_{i=1,...,n}X_i$.
 			
 			For every $\ep >0$, \vspace{-2mm}
 			\beqrs
 			P(|X_{(n)}-1|\geq \ep ) &=& P(X_{(n)} \leq 1-\ep )+P(X_{(n)} \geq 1+\ep ) \\
 			& = & P(X_{(n)} \leq 1-\ep ) \\
 			& = & P(X_i \leq 1-\ep, i=1,...,n ) \\
 			& = & (1-\ep )^n
 			\eeqrs
 			Hence, $X_{(n)}$ converges in probability to 1.
 			
 			In fact, since
 			$\sum_{n=1}^\infty (1-\ep )^n < \infty$, $X_{(n)}$ converges almost surely
 			to 1.
 			
 			For any $t>0$, \vspace{-2mm}
 			\beqrs
 			P(n(1- X_{(n)}) \leq t ) & = & 1-
 			P(n(1- X_{(n)}) \geq t ) \\
 			& = & 1- P(X_{(n)} \leq 1-t/n ) \\
 			& = & 1- (1-t/n)^n \\
 			& \rightarrow & 1-e^{-t}
 			\eeqrs
 			which is the cdf of the $exponential(0,1)$ distribution.
 		\end{alertblock}}
 		\frame{\vspace{-2mm}
 			\begin{alertblock}{ }
 				It is clear that $P(n(1- X_{(n)}) \leq t ) =0$ if $t \leq 0$.
 				
 				Thus, $n(1-X_{(n)})$ converges in distribution to $Z \sim exponential(0,1)$.
 			\end{alertblock}\vspace{-2mm}
 			\begin{block}{}
 				The next theorem shows that convergence in distribution is weaker than convergence in probability
 				and, hence, is also weaker than almost sure convergence.
 			\end{block}\vspace{-2mm}
 			\begin{block}{Theorem 5.5.12.}
 				If $Z_n$ converges in probability to $Z$, then $Z_n$ converges in distribution to $Z$.
 			\end{block}\vspace{-2mm}
 			\begin{block}{Proof.}
 				For any $x \in {\cal R}$ and $\ep > 0$,  \vspace{-2mm}
 				$$\begin{aligned}
 				F_{Z}(x-\epsilon)  & =  P\big( Z \leq x-\epsilon \big) \\
 				& \leq   P\big(  Z_n \leq x \big)
 				+  P\big( Z \leq x-\epsilon ,  Z_n > x \big) \\
 				& \leq F_{Z_n} (x) +  P\left( |Z_n - Z |> \epsilon \right) .
 				\end{aligned} \vspace{-2mm}$$
 				Letting $n \rightarrow \infty$, we obtain that \vspace{-2mm}
 				\[ F_Z ( x - \epsilon ) \leq \liminf_n F_{Z_n}(x).\]
 			\end{block}}
 			\frame{
 				\begin{block}{}
 					Switching $Z_n$ and $Z$ in the previous argument, we can show that\vspace{-2mm}
 					\[ F_Z ( x + \epsilon ) \geq \limsup_n F_{Z_n}(x)\vspace{-2mm}\]
 					i.e., \vspace{-2mm}
 					\[ F_Z ( x - \epsilon ) \leq \liminf_n F_{Z_n}(x) \leq  \limsup_n F_{Z_n}(x) \leq F_Z ( x + \epsilon ) \vspace{-2mm} \]
 					Since $\epsilon$ is arbitrary, \vspace{-2mm}
 					\[ \lim_{\ep \rightarrow 0} F_Z ( x - \epsilon ) \leq \liminf_n F_{Z_n}(x) \leq  \limsup_n F_{Z_n}(x) \leq \lim_{\ep \rightarrow 0}
 					F_Z ( x + \epsilon ) \vspace{-2mm} \]
 					Now, if $F_Z$ is continuous at $x$, then the limit on the far left hand side equals the limit on the far right hand side and both are equal to $F_Z(x)$,
 					which shows that  \vspace{-2mm}
 					\[ F_Z(x)= \lim_{n\rightarrow \infty} F_{Z_n}(x). \]
 				\end{block}\vspace{-2mm}
 				\begin{alertblock}{Example.}
 					The converse of Theorem 5.5.12 is not true in general.
 					\vspace{2mm}
 					
 					Let $\theta_n = 1+n^{-1}$ and $X_n$ be a random variable having the
 					$exponential(0,\theta_n)$ distribution, $n=1,2,...$.
 					
 					Let $X$ be a random variable $\sim  exponential(0,1)$.
 				\end{alertblock}
 			}
 			\frame {\vspace{-2mm}
 				\begin{alertblock}{}
 					For any $x>0$, as $n \rightarrow \infty$, \vspace{-2mm}
 					\[ F_{X_n}(x) = 1-e^{-x/\theta_n}  \rightarrow 1-e^{-x} = F_X(x) \vspace{-3mm} \]
 					Since $F_{X_n}(x)\equiv 0 \equiv F_X(x)$ for
 					$x \leq 0$, we have shown that $ X_n $ converges in distribution to $X$.
 					\vspace{2mm}
 					
 					Does $X_n $ converge in probability to $X$?
 					\begin{itemize}
 						\item Need further information about the random variables $X$ and $X_n$.
 						\item We consider two cases in which different answers can be obtained.
 					\end{itemize}
 				\end{alertblock}\vspace{-2mm}
 				\begin{overprint}
 					\begin{block}{Case 1}
 						Suppose that $X_n \equiv \theta_n X$ (then $X_n$ has the given distribution).
 						
 						$X_n-X = (\theta_n -1) X = n^{-1}X$, which has the cdf
 						$ (1- e^{-nx})I_{[0,\infty )}(x)$.
 						
 						Then,  $X_n$ converges in probability to $ X$, because, for any $\epsilon >0$, \vspace{-2mm}
 						\[ P \left(  |X_n -X| \geq \epsilon  \right)
 						= e^{-n \epsilon } \rightarrow 0 \vspace{-2mm} \]
 						In fact, $X_n$ converges almost surely to $ X$, since \vspace{-3mm}
 						\[ \sum_{n=1}^\infty e^{-n \ep} <\infty \]
 					\end{block}
 				\end{overprint}
 			}
 			
 			\frame {\vspace{-2mm}
 				\begin{block}{Case 2}
 					Suppose that $X_n$ and $X$ are independent random variables.
 					
 					Since the pdf's of $X_n$ and
 					$-X$ are $\theta_n^{-1}e^{-x/\theta_n}I_{(0,\infty )}(x)$
 					and $e^{x}I_{(- \infty,0 )}(x)$, respectively,
 					we have \vspace{-3mm}
 					\[P\left(  |X_n-X| \leq \epsilon  \right) =
 					\int_{-\epsilon }^{\epsilon} \int \theta_n^{-1} e^{-x/\theta_n}
 					e^{y-x} I_{(0,\infty )}(x) I_{(-\infty , x)}(y) dx dy ,\vspace{-3mm} \]
 					which converges to (by the dominated convergence theorem)\vspace{-3mm}
 					\[\int_{-\epsilon}^{ \epsilon} \int e^{-x}
 					e^{y-x} I_{(0,\infty )}(x) I_{(-\infty , x)}(y) dx dy
 					= 1- e^{-\epsilon }. \vspace{-2mm}\]
 					Thus,\vspace{-2mm}
 					\[ P\left(  |X_n-X| \geq \epsilon  \right)
 					\rightarrow e^{-\epsilon } >0\]
 					for any $\epsilon >0$ and, therefore,
 					$X_n $ does not converge in probability to $X$.
 				\end{block}\vspace{-2mm}
 				\begin{alertblock}{}
 					In one situation, however, convergence in distribution is equivalent to
 					convergence in probability, as the following result shows.
 				\end{alertblock}\vspace{-2mm}
 				\begin{alertblock}{Theorem 5.5.13.}
 					$Z_n$ converges in probability to a constant $c$ iff $Z_n$ converges in distribution to $c$.
 				\end{alertblock}
 			}
 			\frame{\vspace{-2mm}
 				\begin{alertblock}{Proof.}
 					The ``only if" part is a special case of Theorem 5.5.12.
 					
 					Hence, we only need to show the ``if" part.
 					
 					If $Z_n$ converges in distribution to a constant $c$, then \vspace{-3mm}
 					\[ \lim_{n\rightarrow \infty} P(Z_n \leq x ) = \left\{ \br{ll}
 					0 & \quad x < c \vspace{1mm}\\
 					1 & \quad x> c \er \right. \vspace{-3mm} \]
 					which is the ``cdf'' of a constant $c$. (Note that the limit does not include the case of $x=0$, which is a discontinuity point of
 					the cdf of $c$.
 					
 					For every $\ep >0$, \vspace{-2mm}
 					\beqrs
 					P(|Z_n-c| \geq \ep ) &=& P(Z_n-c \geq \ep ) + P(Z_n-c \leq - \ep ) \\
 					& = &  P(Z_n \geq c+ \ep )+ P(Z_n \leq c- \ep ) \\
 					& = &  1- P(Z_n < c+ \ep )+ P(Z_n \leq c- \ep ) \\
 					& \leq & 1- P(Z_n < c+ \ep/2 )+ P(Z_n \leq c- \ep ) \\
 					& \rightarrow & 1 -1 + 0 = 0
 					\eeqrs
 					since $c-\ep < c$ and $c+ \ep /2 >c$.
 					
 					This proves that $Z_n$ converges in probability to $c$.
 				\end{alertblock}}
 				
\end{document}


