% $Header: /cvsroot/latex-beamer/latex-beamer/examples/beamerexample5.tex,v 1.22 2004/10/08 14:02:33 tantau Exp $

\documentclass[11pt]{beamer}
\usepackage{amscd}

%\usetheme{Madrid} % Darmstadt, Boadilla


\mode<presentation> {
  \usetheme{Madrid}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsbsy,alltt,amsmath,times}
\usepackage{epsfig, color, graphicx, rotate, colordvi}
%\usepackage{fancybox,fancyhdr,times,subfigure}
\usepackage{hyperref, mathptmx}


\setbeamercovered{dynamic}

\newcommand{\Lang}[1]{\operatorname{\text{\textsc{#1}}}}

\newcommand{\Class}[1]{\operatorname{\mathchoice
  {\text{\sf \small #1}}
  {\text{\sf \small #1}}
  {\text{\sf #1}}
  {\text{\sf #1}}}}

\newcommand{\NumSAT}      {\text{\small\#SAT}}
\newcommand{\NumA}        {\#_{\!A}}

\newcommand{\barA}        {\,\bar{\!A}}

\newcommand{\Nat}{\mathbb{N}}
\newcommand{\Set}[1]{\{#1\}}

\newtheorem{thm}{Result}%[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{defn}[thm]{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{rem}{Remark}

\newcommand{\cred}{\textcolor {red}}
\newcommand{\cblue}{\textcolor {blue}}
\newcommand{\cgreen}{\textcolor {green}}

\def\ul{\underline}

\def\sign{\;{\rm sign}}
\def\rank{\;{\rm rank}}
\def\disp{\displaystyle}
\def\sn{\sqrt{n}}
\def\Pr{\mathrm{P}}

\def\Cov{\;{\rm Cov}}
\def\L{\underline L}
\def\LL{\overline L}

\def\balpha{{\boldsymbol{\alpha}}}
\def\cal{\mathcal}
\def\conas{\stackrel{a.s.} {\longrightarrow}}         % conv. a.s.
\def\conp{\stackrel{\cal P} {\longrightarrow}}        % conv. in probability
\def\conD{\stackrel{\mathcal{L}} {\longrightarrow}}        % conv. in distribution
\def\conw{\stackrel{\mathcal{L}} {\rightarrow}}        % conv. in distribution
\def\hat{\widehat}
%\def\endproof{\begin{flushright}$\lozenge$\end{flushright}}
\def\Proof{\noindent{\it Proof }}
\def\endproof{\hfill{$\Box$}}
\newcommand{\wcon}{\stackrel{\mathcal{L}}{\longrightarrow}}
\def\Var{\mbox{var}}
\def\br{\begin{array}}
\def\er{\end{array}}
\def\ep{\epsilon}
%%%%%%%%%%%%%%% MY DEFINITIONS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Put your own definitions
%
\def\VaR{{\rm VaR}}
\def\E{{\rm E}}
\def\beqrs{\begin{eqnarray*}}           %\beqrs
\def\eeqrs{\end{eqnarray*}}             %\eeqrs

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\pgfdeclaremask{tu}{beamer-tu-logo-mask}
\pgfdeclaremask{computer}{beamer-computer-mask}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerimage}{beamer-computer}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerworkingimage}{beamer-computerred}
\pgfdeclareimage[mask=tu,height=.5cm]{logo}{beamer-tu-logo}

\logo{\pgfuseimage{logo}}

\title[Stat 609 Lecture 21]{Stat 609: Mathematical Statistics \\ Lecture 21}
\author[UW-Madison]{Jun Shao}
\institute[Statistics]{%
%  Department of Statistics\\
%  University of Wisconsin\\
%  Madison, WI 53706, USA\\
%~~~\\
  }
\date[2015]{}

\colorlet{redshaded}{red!25!bg}
\colorlet{shaded}{black!25!bg}
\colorlet{shadedshaded}{black!10!bg}
\colorlet{blackshaded}{black!40!bg}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\def\radius{0.96cm}
\def\innerradius{0.85cm}

\def\softness{0.4}
\definecolor{softred}{rgb}{1,\softness,\softness}
\definecolor{softgreen}{rgb}{\softness,1,\softness}
\definecolor{softblue}{rgb}{\softness,\softness,1}

\definecolor{softrg}{rgb}{1,1,\softness}
\definecolor{softrb}{rgb}{1,\softness,1}
\definecolor{softgb}{rgb}{\softness,1,1}

\newcommand{\Bandshaded}[2]{
  \color{shadedshaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\Band}[2]{
  \color{shaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\BaenderNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{input tapes}{}}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{Zwei Eingabebänder}{}}
  \pgfputat{\pgfxy(0,4.25)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \ignorespaces}

\newcommand{\BaenderHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{input tapes}{}}
  \color{shaded}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Bandshaded{}{}}

  \color{blackshaded}
  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}
\def\Phic{\mathnormal{\Phi}}

\newcommand{\BaenderZweiHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{Zwei Eingabebänder}{}}%
  \color{blackshaded}
  \pgfputat{\pgfxy(0,4.25)}{\Bandshaded{}{}}
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}%

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \ignorespaces}

\newcommand{\Slot}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.0)}%
    \pgfbezier{\pgfxy(0.1,3.95)}{\pgfxy(0.05,3.9)}{\pgfxy(0,3.9)}%
    \pgfbezier{\pgfxy(-0.05,3.9)}{\pgfxy(-0.1,3.95)}{\pgfxy(-0.1,4.0)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\SlotZwei}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.25)}%
    \pgfbezier{\pgfxy(0.1,4.25)}{\pgfxy(0.05,4.15)}{\pgfxy(0,4.15)}%
    \pgfbezier{\pgfxy(-0.05,4.15)}{\pgfxy(-0.1,4.2)}{\pgfxy(-0.1,4.25)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\ClipSlot}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4cm}}}{\pgfxy(0.2,1.5)}\ignorespaces}

\newcommand{\ClipSlotZwei}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4.25cm}}}{\pgfxy(0.2,1.25)}\ignorespaces}


\AtBeginSection[]{\frame{\frametitle{Outline}\tableofcontents[current]}}

\begin{document}

%\frame{\titlepage}

\frame {\vspace{-2mm}
\frametitle{Lecture 21: Convergence of transformations and generating a random variable}
\vspace{-1mm}
    \begin{alertblock}{}
If $Z_n$ converges to $Z$ in some sense, we often need to check whether
$h(Z_n) $ converges to $h(Z)$ in the same sense.
     \end{alertblock}\vspace{-2mm}
         \begin{block}{Continuous mapping theorem.}
If a sequence of random vectors $Z_n$ converges in distribution to $Z$ and $h$ is a function continuous on $A$ with 
$P_Z(A) =1$, 
then $h(Z_n)$ converges in distribution to $h(Z)$.
\end{block}\vspace{-3mm}
\begin{alertblock}{}
\begin{itemize}
\item
If random variables $Z_n $, $n=1,2,...,$, converges in distribution to $Z \sim N(0,1)$, then
$Z_n^2$ converges in distribution to $Z^2 \sim $ the chi-square distribution with degree of freedom 1.
\item
If $(Z_n,Y_n)$ converges in distribution to $(Z,Y)$ as random vectors, then
$Y_n+Z_n $ converges in distribution to $Y+Z$, $Y_nZ_n$ converges in distribution
to $YZ$, and $Y_n/Z_n $ converges in distribution to $Y/Z$.
\item
In the previous situation if $Z$ and $Y$ are iid $\sim N(0,1)$, then
$Z_n/Y_n $ converges in distribution to the $Cauchy(0,1)$ distribution.
\end{itemize}
\end{alertblock}}
\frame{\vspace{-2mm}
    \begin{alertblock}{}
\begin{itemize}
\item
In the previous situation if $Z$ and $Y$ are iid $\sim N(0,1)$, then
$\max\{Z_n,Y_n\} $ converges in distribution to $ \max \{Z,Y\}$, which has the cdf
$[\Phic (x)]^2$ ($\Phic (x)$ is the cdf of $N(0,1)$).
\item
The condition that
$(Z_n,Y_n)$ converges in distribution to $(Z,Y)$ cannot be relaxed to
$Z_n$ converges in distribution to $Z$
and  $Y_n$ converges in distribution to $Y$,
i.e., we need the convergence of the joint cdf of $(Z_n,Y_n)$; e.g.,
 $Z_n+Y_n$ may not converge in distribution to $Z+Y$ if we only have the marginal convergence.
\item
The next result, which plays an important role in
statistics, establishes the convergence in distribution of
$Z_n+Y_n$,  $Z_nY_n$, or $Z_n/Y_n$ with no
information regarding the joint cdf of $(Z_n,Y_n)$.
 \end{itemize}
\end{alertblock}\vspace{-2mm}
\begin{block}{Theorem 5.5.17 (Slutsky's theorem)}
Let $X_n$ converges in distribution to $X$ and  $Y_n$ converges in distribution (probability) to  $c$ (a constant),
then
\begin{itemize}
  \item[(i)] $X_n+Y_n $ converges in distribution to $X+c$;
  \item[(ii)] $Y_n X_n $ converges in distribution to $ cX$;
  \item[(iii)] $X_n/Y_n $ converges in distribution to $ X/c$ if $c \neq 0$.
\end{itemize}
\end{block}}
\frame{
 \begin{block}{Proof.}
We prove (i) only. (The proofs of (ii) and (iii) are left as exercises.)

For $t \in {\cal R}$ and $\epsilon >0$ being fixed constants, \vspace{-2mm}
$$
\begin{aligned}
F_{X_n+Y_n}(t)  & =  P(X_n+Y_n\leq t) \\
& \leq  P( \{ X_n+Y_n\leq t \} \cap \{ |Y_n-c| < \epsilon \})
+ P( |Y_n-c| \geq \epsilon ) \\
& \leq  P( X_n\leq t -c + \epsilon )
+ P( |Y_n-c| \geq \epsilon ) \vspace{-2mm}
\end{aligned}
$$
Similarly,\vspace{-2mm}
\[ F_{X_n+Y_n} (t) \geq P(X_n \leq t-c -\epsilon ) -
P( |Y_n-c| \geq \epsilon ).\vspace{-2mm} \]
If $t-c$, $t-c+\epsilon$, and $t-c-\epsilon$ are continuity
points of $F_X$, then it follows from the previous two inequalities
and the convergence of $X_n+Y_n$ that \vspace{-2mm}
\[ F_X(t-c-\epsilon ) \leq \liminf_n F_{X_n+Y_n}(t) \leq
\limsup_n  F_{X_n+Y_n}(t) \leq
F_X(t-c+\epsilon ) .\vspace{-3mm} \]
Since $\epsilon$ can be arbitrary (why?),
\[ \lim_{n\rightarrow \infty} F_{X_n+Y_n}(t) = F_X(t-c). \vspace{-3mm}\]
The result follows from $F_{X+c}(t) = F_X(t-c)$.
\end{block}}
\frame { \frametitle{}
\begin{alertblock}{Example 5.5.18.}
A typical application of Slutsky's theorem is in establishing the normal approximation with estimated variance.
\vspace{1mm}

For a random sample $X_1,...,X_n$  with finite $\mu = E(X_1)$ and variance $\sigma^2 = {\rm Var}(X_1)$, the CLT shows that \vspace{-2mm}
\[ \frac{\sqrt{n}(\bar{X}-\mu)}{\sigma} \mbox{ converges in distribution to } N(0,1) \vspace{-2mm} \]
If $\sigma^2$ is ``estimated" by the sample variance $S^2$, then by Example 5.5.3, $S^2$ converges in probability (distribution) to $\sigma^2$,
and thus $S$ converges in probability to $\sigma$.
\vspace{1mm}

Then Slutsky's theorem shows that \vspace{-2mm}
\[ \frac{\sqrt{n}(\bar{X}-\mu)}{S} \mbox{ converges in distribution to } N(0,1)  \]
\end{alertblock} \vspace{-2mm}
 \begin{block}{}
     Another typical application of Slutsky's theorem is given in the proof of the following
important result.
     \end{block}}
\frame{
\begin{block}{Theorems 5.5.28 (the delta method)}
Let $ X_1,X_2,...$ and $Y$ be random $k$-vectors satisfying \vspace{-2mm}
\[ a_n(X_n-c) \mbox{ converges in distribution to } Y,\vspace{-2mm}\]
where $c \in {\cal R}^k$ is a constant vector  and $ a_n$'s are
 positive constants with $\lim_{n\rightarrow \infty} a_n =
\infty$. (In many cases, $c = E(X_n)$ and $a_n = \sqrt{n}$.)

If $g$ is a function from ${\cal R}^k$ to ${\cal R}$ and
is differentiable at $c$, then\vspace{-2mm}
\[  a_n[g(X_n)-g(c)] \mbox{ converges in distribution to }
  \nabla g(c)' Y ,\vspace{-2mm} \]
where $\nabla g(x)' $ is the transpose of the $k$-dimensional vector of partial derivatives of $g$ at
$x$.
In particular, if $Y \sim N(0, \Sigma )$, then \vspace{-2mm}
\[ a_n[g(X_n)-g(c)]  \mbox{ converges in distribution to } N\left(0,
\nabla g(c)' \Sigma \nabla g(c) \right) .  \]
\end{block}\vspace{-2mm}
\begin{alertblock}{}
We only give a proof for the univariate case ($k=1$), in which\vspace{-2mm}
\[  a_n[g(X_n)-g(c)] \mbox{ converges in distribution to }
  g'(c) Y ,\vspace{-2mm} \]
  and if $Y \sim N(0,\sigma^2)$, \vspace{-2mm}
\[  a_n[g(X_n)-g(c)] \mbox{ converges in distribution to }
  N(0, [g'(c)]^2\sigma^2) \]
  \end{alertblock}
}

\frame {
\begin{block}{Proof of Theorem 5.5.28 with $k=1$.}
Let \vspace{-2mm}
\[ Z_n =  a_n[g(X_n)-g(c)] -
a_n g'(c) (X_n-c). \vspace{-2mm}\]
If we can show that $Z_n$ converges to in probability  to 0, then the result follows from  \vspace{-2mm}
\[   a_n[g(X_n)-g(c)] = a_n g'(c) (X_n-c) + Z_n , \vspace{-2mm}\]
the condition that $a_n(X_n-c)$ converges in distribution to $ Y$, and Slutsky's theorem.

The differentiability of $g$ at $c$ means that \vspace{-2mm}
\[ \lim_{x \rightarrow c} \left| \frac{g(x)-g(c) - g'(c) (x-c)}{x-c} \right| =
\lim_{x \rightarrow c} \left| \frac{g(x)-g(c)}{x-c} - g'(c) \right| = 0 \vspace{-2mm}\]
i.e., for any $\epsilon >0$,
there is a $\delta_\epsilon >0$ such that \vspace{-2mm}
\[ | g(x)-g(c)- g'(c)(x-c)| \leq \epsilon | x- c | \quad \quad \mbox{when $|x-c| < \delta_\epsilon$.} \vspace{-2mm} \]
Then, for a fixed $\eta >0$, \vspace{-2mm}
\begin{align*}
	 P( |Z_n| \geq \eta ) & = P( |Z_n| \geq \eta , |X_n-c| \geq \delta_\epsilon  ) 
	 + P( |Z_n| \geq \eta , |X_n-c| < \delta_\epsilon  ) \\
	 & \leq 	 P( |X_n-c| \geq \delta_\epsilon )
+ P( a_n | X_n-c| \geq \eta /\epsilon ,  |X_n-c| < \delta_\epsilon  ) 
\end{align*}
\end{block}}
\frame{
\begin{block}{}
Since $a_n \rightarrow \infty$, $a_n(X_n-c) $ converges in distribution to $Y$ and Slutsky's theorem imply
that $X_n-c$ converges in probability to $0$.
\vspace{2mm}

By the continuity mapping theorem, $a_n(X_n-c)$ converges in distribution to $ Y$ implies
$a_n | X_n-c| $ converges in distribution to $| Y |$ (the function $|x|$ is continuous).
\vspace{2mm}

Without loss of generality, we
assume that  $F_{|Y|}$ is continuous at $\eta/\epsilon$.
\vspace{2mm}

Then
$$\begin{aligned}
 \limsup_n P( |Z_n| \geq \eta ) & \leq
 \lim_{n\rightarrow \infty} P( |X_n-c| \geq \delta_\epsilon ) +   \lim_{n\rightarrow \infty} P( a_n | X_n-c| \geq \eta /\epsilon ) \\
& =  P( | Y | \geq \eta / \epsilon ) .
\end{aligned}$$
and $Z_n$ converges in probability to $ 0$ since $\epsilon$ can be arbitrary.
\end{block}\vspace{-2mm}
    \begin{alertblock}{}
In statistics, we often need a nondegenerated limiting distribution
of $a_n[g(X_n)-g(c)]$ so that probabilities involving
$a_n[g(X_n)-g(c)]$ can be approximated by
the cdf of $\nabla g(c)' Y$, using the delta method.
     \end{alertblock}
}

\frame{
\begin{alertblock}{}
In some situations, however, $\nabla g(c)$ so that the limiting distribution of
$a_n[g(X_n)-g(c)]$ becomes degenerated, in which cases we need to consider a high order delta method.
\end{alertblock}\vspace{-2mm}
\begin{block}{Theorem 5.5.26 ($m$th order delta method)}
Let $ X_n$ and $Y$ be random $k$-vectors and $c$ and $a_n$ be constants satisfying the conditions in Theorem 5.5.28.

 Suppose that $g$ has continuous partial derivatives of order $m>1$
in a neighborhood of $c$, with all the partial derivatives of order
$j$, $1 \leq j \leq m-1$, vanishing at $c$,
but with the $m$th-order partial derivatives not all vanishing at
$c$.
Then\vspace{-2mm}
\[  a_n^{m}[g(X_n)-g(c)] \mbox{ converges in distribution to } \vspace{-2mm} \]
\[  \frac{1}{m!}
\sum_{i_1=1}^k\cdots \sum_{i_m=1}^k \frac{\partial^m g}{\partial
x_{i_1}\cdots \partial x_{i_m}} \bigg|_{x=c} Y_{i_1}\cdots
Y_{i_m} \vspace{-2mm}\]
where $Y_j$ is the $j$th component of $Y$.
When $k=1$, \vspace{-2mm}
\[  a_n^{m}[g(X_n)-g(c)] \mbox{ converges in distribution to } \frac{g^{(m)}(c)}{m!} Y^m   \]
\end{block}
}
\frame{
   \begin{alertblock}{Example.}
       Let $\{ X_n \}$ be a sequence of random variables satisfying\vspace{-2mm}
\[ \sqrt{n}(X_n-c)   \mbox{ converges in distribution to } Z \sim N(0,1).\vspace{-2mm}\]
Consider the function $g(x)=x^2$, $g'(x) = 2x$.

If $c \neq 0$, then
an application of the delta method gives that\vspace{-2mm}
\[ \sqrt{n}(X_n^2-c^2) \mbox{ converges in distribution to }  \sim  N(0,4c^2).\vspace{-2mm}\]
If $c=0$, $g'(c)=0$ but $g''(c) = 2$.

Hence, an application of the 2nd order delta method  gives that\vspace{-2mm}
\[ nX_n^2  \mbox{ converges in distribution to } Z^2 ,\vspace{-2mm}\]
which has the chi-square distribution with degree of freedom 1.

This result can also be obtained by applying
the continuity mapping theorem.
     \end{alertblock}\vspace{-2mm}
     \begin{block}{}
     The next example involves  2-dimensional random vectors.
     \end{block}
}
\frame { \vspace{-2mm}
    \begin{block}{Example 5.5.27 (ratio estimator)}
Let $Z_n = (X_n,Y_n)$, $n=1,2,...$, be iid bivariate random vectors with finite 2nd order moments.
\vspace{2mm}

Let $\bar{X}_n = n^{-1} \sum_{i=1}^n X_i$, $\bar{Y}_n=n^{-1}\sum_{i=1}^n Y_i$,
$\mu_x = E(X_1)$, $\mu_y = E(Y_1) \neq 0$, $\mu = (\mu_x, \mu_y)$, $\sigma_x^2 = $Var$(X_1)$, $\sigma_y^2 = $Var$(Y_1)$,
$\sigma_{xy} = $Cov$(X_1,Y_1)$.
\vspace{2mm}

By the CLT,\vspace{-2mm}
\[ \sqrt{n} (Z_n - \mu )
\mbox{ converges in distribution to } N \left( 0, \left( \begin{array}{cc}
\sigma_x^2 & \sigma_{xy}\\
\sigma_{xy} & \sigma^2_y \end{array} \right) \right) \]
By the delta method, $g(x,y)= x/y$, $\partial g/\partial x = y^{-1}$, $\partial g /\partial y = -xy^{-2}$
\[ \sqrt{n} \left(\frac{\bar{X}_n}{\bar{Y}_n} - \frac{\mu_x}{\mu_y}\right)
\mbox{ converges in distribution to } N(0, \sigma^2 )\vspace{-2mm}\]
\[\sigma^2 = (\mu_y^{-1} \ \,  - \mu_x\mu_y^{-2})\left( \begin{array}{cc}
\sigma_x^2 & \sigma_{xy}\\
\sigma_{xy} & \sigma^2_y \end{array} \right)
\left( \begin{array}{c}
\mu_y^{-1} \\ -\mu_x\mu_y^{-2} \end{array}\right)
= \frac{\sigma_x^2}{\mu_y^2} - \frac{2\mu_x\sigma_{xy}}{\mu_y^3}
+ \frac{\mu_x^2\sigma_y^2}{\mu_y^4}
\]
     \end{block}
}
\frame{\vspace{-2mm} 
\begin{block}{Example 5.5.27 (more discussions)}
What if $\mu_y =0 $ in the previous discussion? 
\vspace{1mm} 

Since the function $g(x,y) = x/y$ is not differentiable at $y=0$, the delta method does not apply. 
\vspace{1mm}

However, the early discussion says that $\bar{X}_n /\bar{Y}_n $ converges in distribution to $Cauchy(0,1)$ if $\mu_x=\mu_y =0$. 
\vspace{1mm}

By the delta method, we can similarly show that, if at least one of $\mu_x$ and $\mu_y$ is not 0, then \vspace{-2mm} 
\[ \sqrt{n} (\bar{X}_n\bar{Y}_n - \mu_x\mu_y)
\mbox{ converges in distribution to } N(0, \sigma^2 )\vspace{-2mm}\]
with \vspace{-2mm} 
\[\sigma^2 = (\mu_y \ \,  \mu_x ) \left( \begin{array}{cc}
\sigma_x^2 & \sigma_{xy}\\
\sigma_{xy} & \sigma^2_y \end{array} \right)
\left( \begin{array}{c}
\mu_y \\ \mu_x \end{array}\right)
= \mu_y^2 \sigma_x^2 + 2\mu_x\mu_y \sigma_{xy}+ \mu_x^2\sigma_y^2 \vspace{-2mm} \]
If $\mu_x =\mu_y =0$, then an application of the 2nd order delta method or continuous map theorem shows that \vspace{-2mm} 
\[ n \bar{X}_n\bar{Y}_n \mbox{ converges in distribution to } Z_1Z_2 \vspace{-2mm} \]
where $Z_1$ and $Z_2$ are independent, $Z_1 \sim N(0,1)$, and $Z_2 \sim N(0,1)$. 
     \end{block}
}
\frame {\vspace{-2mm}
		\begin{alertblock}{Generating a random variable}
		We study how to generate random variables from a given distribution.
		\vspace{1mm}
		
		This may be useful in applications, or in statistical research when we carry out simulation studies, or to approximate
		integrals that do not have explicit forms.
	\end{alertblock}\vspace{-2mm}
	\begin{block}{Theorem 2.1.10}
		Let $X$ have continuous cdf $F_X$ and $Y= F_X(X)$.
		Then $Y$ is uniformly distributed on $(0,1)$, that is,
		$P(Y \leq y ) = y$, $0<y<1$.
	\end{block}\vspace{-2mm}
	\begin{alertblock}{Direct method}
		If we want to generate a random value $X$ from a cdf $F_X$ whose inverse $F^{-1}_X$ exists, then we need only to generate
		a $Y$ from the uniform distribution on $(0,1)$ and
		let $X = F_X^{-1}(Y)$.
		\vspace{-1mm}
		
		Some cdf's may not have an inverse. 
	\end{alertblock}\vspace{-2mm}
	\begin{block}{The inverse of a cdf}
		If $F_X$ is strictly increasing, then its inverse $F_X^{-1}$ is well defined by \vspace{-3mm}
		\[ F_X^{-1} (y) =x \quad \quad \mbox{iff} \quad \quad F_X(x) = y  \]
	\end{block}}
	\frame{\vspace{-2mm} \begin{block}{}
		If $F_X$ is flat on an interval $(x_1,x_2]$ (see the figure), then we  define \vspace{-2mm}
		\[ F_X^{-1} (y) = \inf \{ x: F_X(x) \geq y \} , \quad \quad 0 \leq y \leq 1 \vspace{-6mm} \]
		\begin{itemize}\vspace{-1mm}
			\item
			$F_X^{-1}$ is increasing in $y$.\vspace{-1mm}
			\item
			$F_X(F_X^{-1}(y)) =y $ when $F_X$ is continuous at $F_X^{-1}(y)$. \vspace{-1mm}
			\item
			$F_X^{-1}(y) \leq x $ iff $y \leq F_X(x)$.
		\end{itemize}
	\vspace{-11cm}
		\begin{figure}[h]
			\begin{center}
				%\caption{Graph of $y=\sin^2(x)$ in Example 2.1.2.}
				\includegraphics[height=6in, width=5in]{img5.pdf}
			\end{center}
		\end{figure}
		\vspace{-4mm}
		
		In part b of the figure, $F_X^{-1}(y) = x_1$.
	\end{block}}\frame{\vspace{-2mm}
	\begin{alertblock}{Proof of Theorem 2.1.10}
		For $0<y<1$, \vspace{-2mm}
		\[ \begin{array}{rll}
		P(Y \leq y ) & = P(F_X(X) \leq y ) & \quad  \\
		&= P(F_X^{-1}(F_X(X)) \leq F_X^{-1}(y)) & \quad  \mbox{($F_X^{-1}$ is increasing)} \\
		& =P(X \leq F_X^{-1}(y)) & \quad  \mbox{($P(F_X^{-1}(F_X(X)) <X)=0$)} \\
		&= F_X(F_X^{-1}(y)) & \quad  \\
		&=  y & \quad  \mbox{(continuity of $F_X$)} \end{array}
		\]
	\end{alertblock}
\vspace{-2mm} 
	\begin{block}{Probability integral transform}
		If $U $ is uniformly distributed on $(0,1)$ and $F$ is a cdf on ${\cal R}$, then the random variable $X = F^{-1}(U)$ has cdf $F$.
		\vspace{1mm}
		
		We use the property $F^{-1}(y) \leq x $ iff $y \leq F(x)$.
		\begin{itemize}
			\item
			If $y \leq F(x)$, then $x \geq \inf \{ t: F(t) \geq y \} = F^{-1}(y)$.
			\item
			If $F^{-1}(y) = \inf \{ t: F(t) \geq y \} \leq x$, then $F(F^{-1}(y)) \leq F(x)$. But\vspace{-2mm}
			\[ F(F^{-1}(y)) \begin{array}{ll}
			= y & \quad \mbox{if $F$ is continuous at $y$} \vspace{1mm} \\
			> y & \quad \mbox{if $F$ has a jump at $y$} \end{array} \]
			
			\vspace{-3mm}
			
			Thus, we must have $y \leq F(F^{-1}(y)) \leq F(x) $.
		\end{itemize}
	\end{block}}
	\frame{\vspace{-2mm}\begin{block}{}
		Then, for any $x \in {\cal R}$, \vspace{-2mm}
		\[ P(X \leq x ) = P(F^{-1}(U) \leq x ) = P(U \leq F(x)) = F(x) \]
		Thus, if we can generate $U \sim uniform(0,1)$ (there are many algorithms for this), then $Y=F^{-1}(U)$ is a random variable from the desired cdf.
		\vspace{1mm}
		
		This method requires  $F^{-1}(u) = \inf \{ x: F(x) \geq u\}$ has an explicit form.
		
	\end{block}\vspace{-2mm}
	\begin{alertblock}{Applying the direct method}
		Suppose that we want to generate random variables from
		$exponential (0,2)$, which has cdf  $F(x) = 1- e^{-x/2}$, $x \geq 0$.
		
		Then $F^{-1}(u) =  -2\log (1-u)$, $0<u<1$.
	
	Using $Y = -2\log (1-U)$ for the 10,000 generated $U$ values, the authors obtained that the average of $Y$ values is
	2.0004 and the sample variance of $Y$ values is 4.0908,  close to $E(Y)= 2$ and Var$(Y)= 4$.
		\end{alertblock}	
\vspace{-2mm}
    \begin{block}{Indirect methods}
    	When no simple transformation is available to apply a direct method, the following indirect method can often provide a solution.
    \end{block}}
    \frame{
 \vspace{-2mm}
 \begin{alertblock}{Theorem 5.6.8 (Accept/Reject Algorithm).}
 	Let $f$ and $g$ be pdf's with $M = \sup_{t \in {\cal R}} \frac{f(t)}{g(t)} < \infty$. 
 	
 	The following algorithm
 	generates a random variable $Y \sim f$:
 	\begin{itemize}
 		\item[(a)]
 		generate independent $U \sim uniform(0,1)$ and $W \sim g$;
 		\item[(b)]
 		if $U \leq M^{-1} f(W)/g(W)$, set $Y = W$; otherwise, return to step (a).
 	\end{itemize}
 \end{alertblock}\vspace{-2mm}
 \begin{alertblock}{Proof.}
 	The generated $Y$ satisfies \vspace{-2mm}
 	\beqrs
 	P(Y \leq y) & = & P\left(W \leq y \bigg| U \leq M^{-1} f(W)/g(W)\right) \\
 	&=& \frac{P \left(W \leq y ,  U \leq M^{-1} f(W)/g(W)\right)}{
 		P \left(U \leq M^{-1} f(W)/g(W)\right)} \\
 	& = & \frac{\int_{-\infty}^y \int_0^{M^{-1}f(w)/g(w)} du g(w)dw }{
 		\int_{-\infty}^\infty \int_0^{M^{-1}f(w)/g(w)} du g(w)dw }\\
 	& = & \frac{\int_{-\infty}^y f(w)dw }{
 		\int_{-\infty}^\infty f(w)dw } = \int_{-\infty}^y f(w)dw
 	\eeqrs
 \end{alertblock}
}
\frame{\vspace{-2mm}
\begin{alertblock}{}
		There are cases, however, where the target pdf $f$ is unbounded and it is difficult to find a pdf $g$ such that
		$M = \sup_{t \in {\cal R}} \frac{f(t)}{g(t)} < \infty$.
		
		In such cases we can consider the following algorithm, which is a special case of the so-called Markov Chain Monte Carlo methods.
	\end{alertblock}\vspace{-3mm}
	\begin{alertblock}{Metropolis Algorithm}
		Let $f$ and $g$ be pdf's. Consider the following algorithm:
		\begin{itemize}
			\item[(0)]
			Generate $V \sim g$ and set $Z_0 = V$.
			
			For $i=1,2,...$,
			\item[(1)]
			Generate independent $U_i \sim uniform(0,1)$ and $V_i \sim g$ and compute \vspace{-2mm}
			\[ \rho_i =  \min \left\{ \frac{f(V_i)g(Z_{i-1})}{g(V_i)f(Z_{i-1})} , 1 \right\} \vspace{-2mm} \]
			\item[(2)]
			Set \vspace{-2mm}
			\[ Z_i = \left\{ \br{ll}
			V_i & \quad U_i \leq \rho_i \\
			Z_{i-1} & \quad U_i > \rho_i \er \right. \vspace{-1mm} \]
		\end{itemize}
		Then, as $ i \rightarrow \infty$, $Z_i$ converges in distribution to $Y$.
	\end{alertblock}
}
\end{document}

