% $Header: /cvsroot/latex-beamer/latex-beamer/examples/beamerexample5.tex,v 1.22 2004/10/08 14:02:33 tantau Exp $

\documentclass[11pt]{beamer}
\usepackage{amscd}

%\usetheme{Madrid} % Darmstadt, Boadilla


\mode<presentation> {
  \usetheme{Madrid}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsbsy,alltt,amsmath,times}
\usepackage{epsfig, color, graphicx, rotate, colordvi}
%\usepackage{fancybox,fancyhdr,times,subfigure}
\usepackage{hyperref, mathptmx}


\setbeamercovered{dynamic}

\newcommand{\Lang}[1]{\operatorname{\text{\textsc{#1}}}}

\newcommand{\Class}[1]{\operatorname{\mathchoice
  {\text{\sf \small #1}}
  {\text{\sf \small #1}}
  {\text{\sf #1}}
  {\text{\sf #1}}}}

\newcommand{\NumSAT}      {\text{\small\#SAT}}
\newcommand{\NumA}        {\#_{\!A}}

\newcommand{\barA}        {\,\bar{\!A}}

\newcommand{\Nat}{\mathbb{N}}
\newcommand{\Set}[1]{\{#1\}}

\newtheorem{thm}{Result}%[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{defn}[thm]{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{rem}{Remark}

\newcommand{\cred}{\textcolor {red}}
\newcommand{\cblue}{\textcolor {blue}}
\newcommand{\cgreen}{\textcolor {green}}

\def\ul{\underline}

\def\sign{\;{\rm sign}}
\def\rank{\;{\rm rank}}
\def\disp{\displaystyle}
\def\sn{\sqrt{n}}
\def\Pr{\mathrm{P}}

\def\Cov{\;{\rm Cov}}
\def\L{\underline L}
\def\LL{\overline L}

\def\balpha{{\boldsymbol{\alpha}}}
\def\cal{\mathcal}
\def\conas{\stackrel{a.s.} {\longrightarrow}}         % conv. a.s.
\def\conp{\stackrel{\cal P} {\longrightarrow}}        % conv. in probability
\def\conD{\stackrel{\mathcal{L}} {\longrightarrow}}        % conv. in distribution
\def\conw{\stackrel{\mathcal{L}} {\rightarrow}}        % conv. in distribution
\def\hat{\widehat}
%\def\endproof{\begin{flushright}$\lozenge$\end{flushright}}
\def\Proof{\noindent{\it Proof }}
\def\endproof{\hfill{$\Box$}}
\newcommand{\wcon}{\stackrel{\mathcal{L}}{\longrightarrow}}
\def\Var{\mbox{var}}

%%%%%%%%%%%%%%% MY DEFINITIONS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Put your own definitions
%
\def\VaR{{\rm VaR}}
\def\E{{\rm E}}
\def\beqrs{\begin{eqnarray*}}           %\beqrs
\def\eeqrs{\end{eqnarray*}}             %\eeqrs

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\pgfdeclaremask{tu}{beamer-tu-logo-mask}
\pgfdeclaremask{computer}{beamer-computer-mask}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerimage}{beamer-computer}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerworkingimage}{beamer-computerred}
\pgfdeclareimage[mask=tu,height=.5cm]{logo}{beamer-tu-logo}

\logo{\pgfuseimage{logo}}

\title[Stat 609 Lecture 7]{Stat 609: Mathematical Statistics \\ Lecture 7}
\author[UW-Madison]{Jun Shao}
\institute[Statistics]{%
%  Department of Statistics\\
%  University of Wisconsin\\
%  Madison, WI 53706, USA\\
%~~~\\
  }
\date[2015]{}

\colorlet{redshaded}{red!25!bg}
\colorlet{shaded}{black!25!bg}
\colorlet{shadedshaded}{black!10!bg}
\colorlet{blackshaded}{black!40!bg}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\def\radius{0.96cm}
\def\innerradius{0.85cm}

\def\softness{0.4}
\definecolor{softred}{rgb}{1,\softness,\softness}
\definecolor{softgreen}{rgb}{\softness,1,\softness}
\definecolor{softblue}{rgb}{\softness,\softness,1}

\definecolor{softrg}{rgb}{1,1,\softness}
\definecolor{softrb}{rgb}{1,\softness,1}
\definecolor{softgb}{rgb}{\softness,1,1}

\newcommand{\Bandshaded}[2]{
  \color{shadedshaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\Band}[2]{
  \color{shaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\BaenderNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{input tapes}{}}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{Zwei Eingabebänder}{}}
  \pgfputat{\pgfxy(0,4.25)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \ignorespaces}

\newcommand{\BaenderHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{input tapes}{}}
  \color{shaded}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Bandshaded{}{}}

  \color{blackshaded}
  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{Zwei Eingabebänder}{}}%
  \color{blackshaded}
  \pgfputat{\pgfxy(0,4.25)}{\Bandshaded{}{}}
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}%

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \ignorespaces}

\newcommand{\Slot}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.0)}%
    \pgfbezier{\pgfxy(0.1,3.95)}{\pgfxy(0.05,3.9)}{\pgfxy(0,3.9)}%
    \pgfbezier{\pgfxy(-0.05,3.9)}{\pgfxy(-0.1,3.95)}{\pgfxy(-0.1,4.0)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\SlotZwei}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.25)}%
    \pgfbezier{\pgfxy(0.1,4.25)}{\pgfxy(0.05,4.15)}{\pgfxy(0,4.15)}%
    \pgfbezier{\pgfxy(-0.05,4.15)}{\pgfxy(-0.1,4.2)}{\pgfxy(-0.1,4.25)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\ClipSlot}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4cm}}}{\pgfxy(0.2,1.5)}\ignorespaces}

\newcommand{\ClipSlotZwei}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4.25cm}}}{\pgfxy(0.2,1.25)}\ignorespaces}


\AtBeginSection[]{\frame{\frametitle{Outline}\tableofcontents[current]}}

\begin{document}
	\def\back{\!\!\!\!\!}
	
	\frame {\vspace{-1mm}
		\frametitle{Lecture 7: Interchange of integration and limit}\vspace{-2mm}
				\begin{alertblock}{Differentiating under an integral sign}
					\begin{itemize}
						\item
						To study the properties of a chf, we need some technical result. 
						\item
						When can we switch the differentiation and integration?
						\item
						If the range of the integral is finite, this switch is usually valid.
					\end{itemize}
				\end{alertblock}
				\vspace{-2mm}
				\begin{block}{Theorem 2.4.1 (Leibnitz's rule)}
					If $f(x,\theta )$, $a(\theta )$, and $b(\theta )$ are differentiable with respect to $\theta$, then \vspace{-2mm}
					\[ \frac{d}{d \theta} \int_{a(\theta )}^{b(\theta )} f(x,\theta )dx = f(b(\theta ),\theta )b'(\theta ) - f(a(\theta ),\theta )a'(\theta )
					+ \int_{a(\theta )}^{b(\theta )} \frac{\partial f(x,\theta )}{\partial \theta } dx \vspace{-2mm}\]
					In particular, if $a(\theta ) =a$ and $b(\theta )=b$ are constants, then  \vspace{-2mm}
					\[ \frac{d}{d \theta} \int_{a}^{b} f(x,\theta )dx = \int_{a}^{b} \frac{\partial f(x,\theta )}{\partial \theta } dx
					 \]
				\end{block}\vspace{-3mm}
				\begin{alertblock}{}
					\begin{itemize}
						\item
						If the range of integration is infinite, problems can arise.
						\item
						Whether differentiation and integration can be switched is the same as whether limit and integration can be interchanged. %						because differentiation is a special kind of limit.
					\end{itemize}
				\end{alertblock}
			}
			\frame{
				\begin{block}{Interchange of integration and limit}
					Note that \vspace{-2mm}
					\[ \int_{-\infty}^\infty \frac{\partial f(x,\theta )}{\partial \theta } dx =
					\int_{-\infty}^\infty \lim_{\delta \rightarrow 0} \frac{f(x,\theta + \delta )- f(x,\theta )}{\delta} dx \vspace{-2mm} \]
					Hence, the interchange of differentiation and integration means whether this is equal to \vspace{-2mm}
					\[ \frac{d}{d \theta } \int_{-\infty}^\infty f(x,\theta ) dx =
					\lim_{\delta \rightarrow 0}  \int_{-\infty}^\infty \frac{f(x,\theta + \delta )- f(x,\theta )}{\delta} dx  \]
				\end{block}\vspace{-2mm}
				\begin{alertblock}{An example of invalid interchanging integration and limit}
					Consider
					\[ f(x,\delta ) = \left\{ \begin{array}{ll}
					\delta^{-1} & \quad 0 <x < \delta \\
					0 & \quad \mbox{otherwise} \end{array} \right. \]
					Then, $ \lim_{\delta \rightarrow 0} f(x,\delta ) = 0 $ for any $x$ and, hence,
					\[ 0 = \int_{-\infty}^\infty \lim_{\delta \rightarrow 0} f(x,\delta ) dx \neq
					\lim_{\delta \rightarrow 0}  \int_{-\infty}^\infty f(x,\delta ) dx =
					\lim_{\delta \rightarrow 0}  \int_0^\delta  \delta^{-1} dx = 1  \]
				\end{alertblock}
			}
			\frame{
				\begin{alertblock}{}
					We now give some sufficient conditions under which interchanging integration and limit (or differentiation) is valid.
					\vspace{1mm}
					
					The proof of the main result is technical and out of the scope of this course.
				\end{alertblock}
				\vspace{-2mm}
				\begin{block}{Theorem 2.4.2 (Lebesgue's dominated convergence theorem)}
					Suppose that the function $h(x,y)$ is continuous at $y_0$ for each $x$, and there exists a function $g(x)$ satisfying \\
					(i) $|h(x,y)| \leq g(x)$ for all $x$ and $y$ in a neighborhood of $y_0$; \\
					(ii) $\int_{-\infty}^\infty g(x) dx < \infty$. \\
					Then,
					\[ \lim_{y \rightarrow y_0} \int_{-\infty}^\infty h(x,y)dx =
					\int_{-\infty}^\infty \lim_{y \rightarrow y_0}  h(x,y)dx \]
				\end{block}\vspace{-2mm}
				\begin{block}{}
					The key condition in this theorem is the existence of a dominating function $g(x)$ with a finite integral.
					
					Applying Theorem 2.4.2 to the difference
					$[f(x,\theta + \delta )- f(x,\theta )]/\delta$, we obtain the next result.
				\end{block}
			}
			\frame{ \vspace{-2mm}
				\begin{block}{Theorem 2.4.3.}
					Suppose that $f(x,\theta )$ is differentiable at $\theta = \theta_0 $, i.e., \vspace{-1mm}
					\[ \lim_{\delta \rightarrow 0} \frac{f(x,\theta_0 + \delta ) - f(x,\theta_0 )}{\delta } =
					\frac{\partial f(x,\theta )}{\partial \theta }\bigg|_{\theta = \theta_0} \vspace{-1mm} \]
					exists for every $x$, and there exists a function $g(x,\theta_0)$ and a constant $\delta_0>0$ such that\\
					(i) $\big| \frac{f(x,\theta_0 + \delta ) - f(x,\theta_0 )}{\delta }\big| \leq g(x,\theta_0) $ for all $x$ and $|\delta | \leq \delta_0$; \\
					(ii) $\int_{-\infty}^\infty g(x,\theta_0 ) dx < \infty$. \\
					Then \vspace{-1mm}
					\[ \frac{d}{d \theta } \int_{-\infty}^\infty f(x,\theta) dx \bigg|_{\theta = \theta_0} = \int_{-\infty}^\infty
					\frac{\partial f(x,\theta )}{\partial \theta }\bigg|_{\theta = \theta_0}  dx \]
				\end{block}\vspace{-2mm}
				\begin{alertblock}{}
					This result is for $\theta$ at a particular value, $\theta_0$.
					
					Typically $f(x,\theta )$ is differentiable at all $\theta$, and \vspace{-1mm}
					\[ \frac{f(x,\theta_0 + \delta ) - f(x,\theta_0 )}{\delta } = \frac{\partial f(x,\theta )}{\partial \theta }\bigg|_{\theta = \theta_0 + \delta^*(x)} \vspace{-2mm} \]
					for some $\delta^*(x)$ with $|\delta^*(x)| \leq \delta_0$, which leads to the next result.
				\end{alertblock}
			}
			\frame{
				\begin{alertblock}{Corollary 2.4.4.}
					Suppose that $f(x,\theta )$ is differentiable in $\theta$ and there exists a function $g(x,\theta )$ such that \\
					(i) $\big| \frac{\partial f(x,\vartheta )}{\partial \vartheta } \big| \leq g(x, \theta )$ for all $x$ and $\vartheta$ such that $|\vartheta - \theta | \leq \delta_0$; \\
					(ii) $\int_{-\infty}^\infty g(x,\theta ) dx < \infty$ for all $\theta$. \\
					Then \vspace{-1mm}
					\[\frac{d}{d \theta } \int_{-\infty}^\infty f(x,\theta) dx = \int_{-\infty}^\infty
					\frac{\partial f(x,\theta )}{\partial \theta } dx \]
				\end{alertblock}\vspace{-2mm}
				\begin{block}{Example 2.4.5.}
					Let $X$ have the exponential pdf \vspace{-1mm}
					\[ f(x) = \lambda^{-1} e^{-x/\lambda }, \quad x>0 \vspace{-1mm}\]
					where $\lambda >0$ is a constant.
					
					We want to prove a recursion relation\vspace{-1mm}
					\[ E(X^{n+1}) = \lambda E(X^n) + \lambda \frac{d}{d \lambda } E(X^n) , \quad \quad n=1,2,... \vspace{-2mm} \]
					Note that both $E(X^n)$ and $E(X^{n+1})$ are functions of $\lambda$.
				\end{block}
			}
			\frame{\vspace{-3mm}
				\begin{block}{}
					If we could move the differentiation inside the integral, we would have \vspace{-2mm}
					\beqrs
					\frac{d}{d \lambda } E(X^n) &=&
					\frac{d}{d \lambda } \int_0^\infty \frac{x^n}{\lambda} e^{-x/\lambda } dx
					= \int_0^\infty \frac{\partial }{\partial \lambda} \left( \frac{x^n}{\lambda} e^{-x/\lambda } \right) dx\\
					&=& \int_0^\infty \frac{x^n}{\lambda^2}\left(\frac{x}{\lambda }-1\right) e^{-x/\lambda } dx
					= \frac{1}{\lambda^2 } E(X^{n+1}) - \frac{1}{\lambda} E(X^n)
					\eeqrs
					which is the result we want to show.
					
					To justify the interchange of integration and differentiation, we take \vspace{-2mm}
					\[ g(x,\lambda ) = \frac{x^n e^{-x/(\lambda +\delta_0)} }{(\lambda - \delta_0)^2} \left( \frac{x}{\lambda - \delta_0 }+1\right) \vspace{-2mm} \]
					Then \vspace{-2mm}
					\[ \bigg| \frac{\partial }{\partial \xi } \left( \frac{x^n}{\xi } e^{-x/\xi  } \right) \bigg|
					\leq \frac{x^ne^{-x/\xi}}{\xi^2} \bigg|\frac{x}{\xi}+1\bigg| \leq g(x,\lambda ) , \quad |\xi - \lambda | \leq \delta_0 \]
					and we can apply Corollary 2.4.4.
				\end{block}\vspace{-3mm}
				\begin{alertblock}{}
					In the proof of Theorem 2.3.7 (differentiating mgf to obtain moments), we interchanged differentiation and integration without
					justification. \\
					We now provide a proof.
				\end{alertblock}
			}
			\frame{\vspace{-2mm}
				\begin{alertblock}{Completion of the proof of Theorem 2.3.7.}
					Assuming that $M_X(t)$ exists for $t \in [-h,h]$, $h>0$, we want to show \vspace{-2mm}
					\[ \frac{d}{dt} M_X(t) = \frac{d}{dt} \int_{-\infty}^\infty e^{tx} f_X(x)dx
					= \int_{-\infty}^\infty \frac{\partial }{\partial t} e^{tx} f_X(x)dx
					\vspace{-2mm}\]
					It is enough to show (why?)\vspace{-2mm}
					\[ \frac{d}{dt} \int_{0}^\infty e^{tx} f_X(x)dx
					= \int_{0}^\infty \frac{\partial }{\partial t} e^{tx} f_X(x)dx
					\vspace{-2mm}\]
					For $t \in (-h/2, h/2)$ and $x>0$, \vspace{-2mm}
					\[ \bigg|  \frac{\partial }{\partial t} e^{tx} f_X(x) \bigg| =
					\bigg|  x e^{tx} f_X(x) \bigg| \leq x e^{hx/2}f_X(x) = g(x) \vspace{-2mm}\]
					For sufficiently large $x>0$, $x \leq e^{hx/2}$ and, hence, \vspace{-2mm}
					\[ \int_0^\infty g(x) dx \leq \int_0^\infty e^{hx}f_X(x) dx = M_X(h) < \infty \vspace{-2mm}\]
					An application of Corollary 2.4.4 proves the result.
				\end{alertblock}
				\vspace{-2mm}
				\begin{block}{}
					We now complete the proof of Theorem C1 and establish another useful result for inverting a characteristic function.
					\end{block}
			}
										\frame{\vspace{-2mm}
											\begin{block}{Completion of the proof of Theorem C1.}
												We give a proof for the case where $X$ has pdf $f_X(x)$.
												\vspace{1mm}
												
												If we can exchange the differentiation and integration, then \vspace{-3mm}
												\[ \frac{d^r \phi_X(t)}{dt^r} = \frac{d^r }{dt^r} \int_{-\infty}^\infty e^{\dot{\imath}tx} f_X(x) dx
												= \int_{-\infty}^\infty  \frac{d^r }{dt^r} e^{\dot{\imath}tx} f_X(x) dx
												= \int_{-\infty}^\infty  \dot{\imath}^r x^r e^{\dot{\imath}tx} f_X(x) dx \vspace{-3mm} \]
												and the result follows by setting $t=0$ at the beginning and end of the above expression.
												\vspace{1mm}
												
												The exchange is justified by the dominated convergence theorem, since \vspace{-1mm}
												\[ \bigg| \frac{d^r }{dt^r} e^{\dot{\imath}tx} f_X(x)\bigg|
												= \bigg|  \dot{\imath}^r x^r e^{\dot{\imath}tx}  \bigg| f_X(x) \leq |x|^r f_X(x) \vspace{-2mm} \]
												and \vspace{-1mm}
												\[ E|X|^r = \int_{-\infty}^\infty |x|^r f_X(x) dx < \infty  \]
											\end{block}
\vspace{-3mm}
\begin{alertblock}{Theorem C7.}
	Let $\phi (t)$ be a chf satisfying $\int_{-\infty}^\infty |\phi (t)| dt < \infty$. 
	
	Then the corresponding cdf $F$ has derivative \vspace{-3mm}
	\[ F'(x)  =\frac{1}{2\pi} \int_{-\infty}^\infty e^{-\dot{\imath}tx} \phi(t) dt \]
\end{alertblock}										}
										
										\frame{\vspace{-2mm}
											\begin{alertblock}{Proof.}
												Let $x-\delta $ and $x+\delta$ be continuity points of $F$.
												
												By Theorem C2, \vspace{-3mm}
												\beqrs F(x+\delta ) - F(x-\delta ) \back & = & \back
												\lim_{T \rightarrow \infty}
												\int_{-T}^T \frac{e^{-\dot{\imath} t(x-\delta)} -e^{-\dot{\imath} t(x+\delta )} }{2\pi \dot{\imath} t} \phi(t) dt \\
												\back & = & \back \lim_{T \rightarrow \infty}
												\int_{-T}^T \frac{\sin (t\delta )e^{-\dot{\imath} tx} }{\pi t} \phi(t) dt \\
												\back & = & \back \int_{-\infty}^\infty \frac{\sin (t\delta )e^{-\dot{\imath} tx} }{\pi t} \phi(t) dt
												\eeqrs
												$~~$ \vspace{-6mm}
												
												where the last equality follows from the fact that \vspace{-3mm}
												\[ \bigg| \frac{\sin (t\delta )e^{-\dot{\imath} tx} }{\pi t\delta } \phi(t) \bigg| \leq | \phi(t)|  \]
												which is integrable.
												\vspace{1mm}
												
												Also, from the previous bound we can apply the dominated convergence theorem 
												to interchange the integration and limit in the following operation to finish the proof:									
													\end{alertblock}
										}
										
										\frame{\vspace{-2mm}
											\begin{alertblock}{}
												\vspace{-4mm}
												\beqrs
												F'(x) \back & = & \back \lim_{\delta \rightarrow 0} \frac{F(x+\delta ) - F(x-\delta )}{2 \delta} \\
\back & = & \back \lim_{\delta \rightarrow 0}\int_{-\infty}^\infty \frac{\sin (t\delta )e^{-\dot{\imath} tx} }{2 \pi t\delta } \phi(t) dt \\
												\back & = & \back
												\int_{-\infty}^\infty \lim_{\delta \rightarrow 0} \frac{\sin (t\delta )e^{-\dot{\imath} tx} }{2 \pi t\delta } \phi(t) dt\\
\back & = & \back \frac{ 1}{2 \pi }  \int_{-\infty}^\infty e^{-\dot{\imath} tx} \phi(t) dt
												\eeqrs
											\end{alertblock}
		\vspace{-3mm} 
		\begin{alertblock}{Example}
			We now apply Theorem C7 to obtain the pdf corresponding to the chf $\phi(t) = e^{-|t|}$:\vspace{-2mm} 
			\[ F'(x) =  \frac{ 1}{2 \pi }  \int_{-\infty}^\infty e^{-\dot{\imath} tx} e^{-|t|} dt
			=  \frac{ 1}{ \pi }  \int_{0}^\infty \cos (tx) e^{-t} dt \vspace{-2mm} \]
			Since ${\displaystyle \int} e^{at} \cos (bt) dt = e^{at} [ a\cos (bt) + b \sin (bt) ]/(a^2+b^2) $,  
		\vspace{-3mm} 
			\[ F'(x) =  \frac{ 1}{ \pi } \frac{e^{-t} [ - \cos (xt) + x \sin (xt) ]}{1+x^2} \bigg|_{t=0}^{t= \infty} = \frac{1}{\pi (1+x^2)} \]
		\end{alertblock}}
		\frame{\vspace{-2mm}
								\begin{block}{}
									We now turn to the question of when we can interchange differentiation and infinite summation.
								\end{block}
								\vspace{-3mm}
				\begin{block}{Theorem 2.4.8.}
					Suppose that the series $\sum_{x=0}^\infty h(\theta , x)$ converges for all $\theta$ in an interval $(a,b) \subset {\cal R}$ and\\
					(i) $\frac{\partial}{\partial \theta } h(\theta , x)$ is continuous in $\theta$ for each $x$; \\
					(ii) $\sum_{x=0}^\infty \frac{\partial}{\partial \theta }h(\theta , x)$ converges uniformly on every closed bounded subinterval of $(a,b)$. \\
					Then\vspace{-2mm}
					\[ \frac{d}{d \theta } \sum_{x=0}^\infty h(\theta , x) = \sum_{x=0}^\infty \frac{\partial}{\partial \theta } h(\theta , x) \]
				\end{block}\vspace{-2mm}
				\begin{alertblock}{}
					This theorem is from calculus (or mathematical analysis) and is not easy to apply because we need to show the uniform convergence of an infinite series.
					\vspace{1.5mm}
					
					It is more convenient to apply the following dominated convergence theorem for infinite sums.
				\end{alertblock}}
				\frame{\vspace{-2mm}
				\begin{block}{Theorem 2.4.2A (dominated convergence theorem)}
					Suppose that the function $h(x,y)$ is continuous at $y_0$ for each $x$, and there exists a function $g(x)$ satisfying
				
						(i) $|h(x,y)| \leq g(x)$ for all $x$ and $y$ in a neighborhood of $y_0$; \\
						(ii) $\sum_x  g(x) < \infty$. \\
						Then, \vspace{-2mm}
						\[ \lim_{y \rightarrow y_0} \sum_x  h(x,y) =
						\sum_x  \lim_{y \rightarrow y_0}  h(x,y)\vspace{-2mm} \]
						If $h(x,\theta )$ is differentiable in $\theta$ and there is a function $g(x,\theta )$ such that \\
						(i) $\big| \frac{\partial h(x,\vartheta )}{\partial \vartheta } \big| \leq g(x, \theta )$ for all $x$ and $\vartheta$ such that $|\vartheta - \theta | \leq \delta_0$; \\
						(ii) $\sum_x g(x,\theta ) < \infty$ for all $\theta$. \\
						Then \vspace{-1mm}
						\[\frac{d}{d \theta } \sum_x h(x,\theta)  = \sum_x
						\frac{\partial h(x,\theta )}{\partial \theta }\vspace{-1mm}  \]
					\end{block}\vspace{-2mm}
					\begin{alertblock}{Example 2.4.7.}
						Let $X$ be a discrete random variable having pmf \vspace{-3mm}
						\[ f_X(x) = P(X=x) = \theta (1-\theta )^x , \quad x=0,1,2,... \vspace{-3mm}\]
						where $\theta \in (0,1)$ is a fixed constant.
						\vspace{1mm}
						
						We want to derive $E(X)$.
					\end{alertblock}
				}
				\frame{\vspace{-2mm} 
					\begin{alertblock}{}
											Note that
						\vspace{-2mm}
						\[ \sum_{x=0}^\infty \theta (1-\theta )^x = 1 		\vspace{-2mm}
						 \]
							If interchange of summation and differentiation is justified,  \vspace{-3mm}
							\beqrs
							0  &  = & \frac{d}{d \theta } \sum_{x=0}^\infty \theta (1-\theta )^x
							=    \sum_{x=0}^\infty  \frac{\partial }{\partial \theta } \theta  (1-\theta )^x \\
							& = &     \sum_{x=0}^\infty  [ (1-\theta )^x - \theta x(1-\theta )^{x-1} ] \\
							& = &    \frac{1}{\theta } \sum_{x=0}^\infty  \theta (1-\theta )^x - \frac{1}{1-\theta } \sum_{x=0}^\infty x\theta  (1-\theta )^{x} \\
							& = & \frac{1}{\theta } - \frac{1}{1-\theta } E(X)
							\eeqrs	
																Thus, $E(X) = (1-\theta )/\theta$.
							\vspace{1mm}
							
							To justify the interchange, let $h(x,\theta ) = \theta (1-\theta )^x$ and then \vspace{-2mm}
							\[ \frac{\partial}{\partial \theta }h(\theta , x) = (1-\theta )^x - \theta x(1-\theta )^{x-1} \vspace{-2mm} \]
							Consider $\theta \in [c,d] \subset (0,1)$. 
						\end{alertblock}
					}
					\frame{\vspace{-2mm} 
						\begin{alertblock}{}\vspace{-2mm}
							\[ | (1-\theta )^x - \theta x(1-\theta )^{x-1} | \leq (1-c)^x + d x(1-c)^{x-1} = g(x)\vspace{-2mm} \]
							Since\vspace{-2mm}
							\[ \sum_{x=0}^\infty [ (1-c)^x + d x(1-c)^{x-1} ] < \infty \vspace{-2mm} \]
							the exchange of sum and differentiation is justified.
						\end{alertblock}
						\vspace{-2mm}
						\begin{block}{Monotone convergence theorem}
							In some cases, the following result is more convenient to apply.
							\vspace{1mm}
							
							Suppose that functions $g_n(x)$, $n=1,2,...$, $x \in {\cal R}$,
							satisfying that $0 \leq g_n(x) \leq g_{n+1}(x)$ for all $n$ and $x$ and $\lim_{n\rightarrow \infty}g_n(x) = g(x)$ for all $x$.
							Then \vspace{-1mm}
							\[ \lim_{n\rightarrow \infty} \int_{-\infty}^\infty g_n(x) dx = \int_{-\infty}^\infty \lim_{n\rightarrow \infty} g_n(x) dx
							= \int_{-\infty}^\infty g(x) dx \vspace{-2mm} \]
							which holds even if $g(x)$ is not integrable (i.e., both sides are infinity).
							
							The same result holds when $\int$ is replaced by $\sum$ for discrete $x$. 
							\end{block}\vspace{-2mm}
					\begin{block}{Proof.}
									We prove the continuous case, since the discrete case is similar.
										\vspace{1mm} 
										
							If $\int_{-\infty}^\infty g(x)dx < \infty$,
							then this result is a direct consequence of the dominated convergence theorem.
				\end{block}
			}
			\frame{\vspace{-2mm} 
				\begin{block}{}
							Suppose now that $\int_{-\infty}^\infty g(x)dx = \infty$.
								
								 Then %\vspace{-2mm}
								\[ \lim_{M \rightarrow \infty} \int_{-\infty}^\infty  g(x)I_{\{|x| \leq M, g(x) \leq M\}} dx = \infty \vspace{-2mm} \]
								where $I_A$ is the indicator function of $A$.
								\vspace{1.5mm}
								
								For each $M>0$, \vspace{-2mm} 
								\[ \int_{-\infty}^\infty  g(x)I_{\{|x| \leq M, g(x) \leq M\}} dx \leq 2M^2 < \infty\vspace{-2mm} \]
								 and
								$g_n(x) I_{\{|x| \leq M, g(x) \leq M\}}$ is monotone and converges to $g(x)I_{\{|x| \leq M, g(x) \leq M\}}$.
							\vspace{1mm}
							
								Then, 	for every $M$.
								\vspace{-2mm}
								\beqrs
								\lim_{n\rightarrow \infty} \int_{-\infty}^\infty g_n(x)  dx \back & \geq & \back \lim_{n\rightarrow \infty} \int_{-\infty}^\infty g_n(x) I_{\{|x| \leq M, g(x) \leq M\}} dx\\
								& = & \back \int_{-\infty}^\infty g(x) I_{\{|x| \leq M, g(x) \leq M\}} dx
								\eeqrs
								$~~$ \vspace{-6mm}
								
								Since the right side goes to $\infty$, \vspace{-2mm}
								\[ \lim_{n\rightarrow \infty} \int_{-\infty}^\infty g_n(x)  dx = \infty \]
							
							\end{block}}
\end{document}
