% $Header: /cvsroot/latex-beamer/latex-beamer/examples/beamerexample5.tex,v 1.22 2004/10/08 14:02:33 tantau Exp $

\documentclass[11pt]{beamer}
\usepackage{amscd}

%\usetheme{Madrid} % Darmstadt, Boadilla


\mode<presentation> {
  \usetheme{Madrid}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsbsy,alltt,amsmath,times}
\usepackage{epsfig, color, graphicx, rotate, colordvi}
%\usepackage{fancybox,fancyhdr,times,subfigure}
\usepackage{hyperref, mathptmx}


\setbeamercovered{dynamic}

\newcommand{\Lang}[1]{\operatorname{\text{\textsc{#1}}}}

\newcommand{\Class}[1]{\operatorname{\mathchoice
  {\text{\sf \small #1}}
  {\text{\sf \small #1}}
  {\text{\sf #1}}
  {\text{\sf #1}}}}

\newcommand{\NumSAT}      {\text{\small\#SAT}}
\newcommand{\NumA}        {\#_{\!A}}

\newcommand{\barA}        {\,\bar{\!A}}

\newcommand{\Nat}{\mathbb{N}}
\newcommand{\Set}[1]{\{#1\}}

\newtheorem{thm}{Result}%[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{defn}[thm]{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{rem}{Remark}

\newcommand{\cred}{\textcolor {red}}
\newcommand{\cblue}{\textcolor {blue}}
\newcommand{\cgreen}{\textcolor {green}}

\def\ul{\underline}

\def\sign{\;{\rm sign}}
\def\rank{\;{\rm rank}}
\def\disp{\displaystyle}
\def\sn{\sqrt{n}}
\def\Pr{\mathrm{P}}

\def\Cov{\;{\rm Cov}}
\def\L{\underline L}
\def\LL{\overline L}

\def\balpha{{\boldsymbol{\alpha}}}
\def\cal{\mathcal}
\def\conas{\stackrel{a.s.} {\longrightarrow}}         % conv. a.s.
\def\conp{\stackrel{\cal P} {\longrightarrow}}        % conv. in probability
\def\conD{\stackrel{\mathcal{L}} {\longrightarrow}}        % conv. in distribution
\def\conw{\stackrel{\mathcal{L}} {\rightarrow}}        % conv. in distribution
\def\hat{\widehat}
%\def\endproof{\begin{flushright}$\lozenge$\end{flushright}}
\def\Proof{\noindent{\it Proof }}
\def\endproof{\hfill{$\Box$}}
\newcommand{\wcon}{\stackrel{\mathcal{L}}{\longrightarrow}}
\def\Var{\mbox{var}}

%%%%%%%%%%%%%%% MY DEFINITIONS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Put your own definitions
%
\def\VaR{{\rm VaR}}
\def\E{{\rm E}}
\def\beqrs{\begin{eqnarray*}}           %\beqrs
\def\eeqrs{\end{eqnarray*}}             %\eeqrs

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\pgfdeclaremask{tu}{beamer-tu-logo-mask}
\pgfdeclaremask{computer}{beamer-computer-mask}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerimage}{beamer-computer}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerworkingimage}{beamer-computerred}
\pgfdeclareimage[mask=tu,height=.5cm]{logo}{beamer-tu-logo}

\logo{\pgfuseimage{logo}}

\title[Stat 609 Lecture 6]{Stat 609: Mathematical Statistics \\ Lecture 6}
\author[UW-Madison]{Jun Shao}
\institute[Statistics]{%
%  Department of Statistics\\
%  University of Wisconsin\\
%  Madison, WI 53706, USA\\
%~~~\\
  }
\date[2015]{}

\colorlet{redshaded}{red!25!bg}
\colorlet{shaded}{black!25!bg}
\colorlet{shadedshaded}{black!10!bg}
\colorlet{blackshaded}{black!40!bg}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\def\radius{0.96cm}
\def\innerradius{0.85cm}

\def\softness{0.4}
\definecolor{softred}{rgb}{1,\softness,\softness}
\definecolor{softgreen}{rgb}{\softness,1,\softness}
\definecolor{softblue}{rgb}{\softness,\softness,1}

\definecolor{softrg}{rgb}{1,1,\softness}
\definecolor{softrb}{rgb}{1,\softness,1}
\definecolor{softgb}{rgb}{\softness,1,1}

\newcommand{\Bandshaded}[2]{
  \color{shadedshaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\Band}[2]{
  \color{shaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\BaenderNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{input tapes}{}}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{Zwei Eingabebänder}{}}
  \pgfputat{\pgfxy(0,4.25)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \ignorespaces}

\newcommand{\BaenderHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{input tapes}{}}
  \color{shaded}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Bandshaded{}{}}

  \color{blackshaded}
  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{Zwei Eingabebänder}{}}%
  \color{blackshaded}
  \pgfputat{\pgfxy(0,4.25)}{\Bandshaded{}{}}
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}%

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \ignorespaces}

\newcommand{\Slot}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.0)}%
    \pgfbezier{\pgfxy(0.1,3.95)}{\pgfxy(0.05,3.9)}{\pgfxy(0,3.9)}%
    \pgfbezier{\pgfxy(-0.05,3.9)}{\pgfxy(-0.1,3.95)}{\pgfxy(-0.1,4.0)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\SlotZwei}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.25)}%
    \pgfbezier{\pgfxy(0.1,4.25)}{\pgfxy(0.05,4.15)}{\pgfxy(0,4.15)}%
    \pgfbezier{\pgfxy(-0.05,4.15)}{\pgfxy(-0.1,4.2)}{\pgfxy(-0.1,4.25)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\ClipSlot}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4cm}}}{\pgfxy(0.2,1.5)}\ignorespaces}

\newcommand{\ClipSlotZwei}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4.25cm}}}{\pgfxy(0.2,1.25)}\ignorespaces}


\AtBeginSection[]{\frame{\frametitle{Outline}\tableofcontents[current]}}
\def\back{\!\!\!\!\!}

\begin{document}

%\frame{\titlepage}

\frame {\vspace{-1mm}
\frametitle{Lecture 6: Characteristic functions and inequalities}
\vspace{-2mm}
\begin{alertblock}{}
The mgf's are useful, but they sometimes are not finite.
\vspace{1mm}

Another function that can characterise a distribution is the so-called characteristic function, which is always well defined but it is a complex function.
\end{alertblock}
\vspace{-2mm}
\begin{block}{Definition (characteristic functions).}
The \cblue{characteristic function} (chf) of the distribution of a random variable $X$  is
defined as\vspace{-2mm}
\[ \phi_X (t) = E (e^{ \dot{\imath}  t  X })
= E[ \cos ( t X)] + \dot{\imath}  E [\sin (tX)] , \quad\quad  t
\in {\cal R} \vspace{-2mm} \]
where $\dot{\imath}  = \sqrt{-1} $ and $e^{ \dot{\imath}  s } = \cos (s) + \dot{\imath} \sin (s)$.
\end{block}
\vspace{-2mm}
\begin{alertblock}{}
\begin{itemize}
\item
If the mgf $M_X(t)$ of $X$ is finite in a neighborhood of
$0$, then the chf of $X$ is $\phi_X (t)  = M_X(\dot{\imath} t)$, $t \in {\cal R}$.
\item
If $Y = aX+b$ for constants $a$ and $b$, then the chf of $Y$ in terms of the chf $\phi_X(t)$ of $X$ is
$ \phi_Y(t) = e^{\dot{\imath}bt} \phi_X(at)$, $t \in {\cal R}$.
\item
A chf is uniformly continuous in ${\cal R}$. 
\end{itemize}
\end{alertblock}
}
\frame{
	\vspace{-2mm}
	\begin{block}{Example.}
		Let $X$ be a random variable with pdf \vspace{-2mm}
		\[ f_X(x) = \frac{1}{\pi (1+x^2)} , \quad \quad x \in {\cal R}. \vspace{-2mm} \]
		When $t>0$, \vspace{-2mm}
		\beqrs
		\phi_X(t) \back & = & \back E[\cos (tX) ] + \dot{\imath} E[ \sin(tX)] \\
		\back & = & \back \int_{-\infty}^\infty \frac{\cos (tx)}{\pi(1+x^2)} dx + \dot{\imath}
		\int_{-\infty}^\infty \frac{\sin (tx)}{\pi(1+x^2)} dx \\
		\back & = & \back \frac{2}{\pi} \int_{0}^\infty \frac{\cos (tx)}{1+x^2} dx = e^{-t}
		\eeqrs
		$~~$ \vspace{-6mm}
		
		since $\sin (s)$ is an odd function and $\cos (s)$ is an even function, where the last equality follows from the Fourier transformation. 
		
		For $t < 0$, \vspace{-2mm}
		\[ \phi_X(t)  =  E[\cos (tX) ] + \dot{\imath} E[ \sin (tX)]
		= E[\cos (-tX) ] = e^t = e^{-|t|} \vspace{-2mm} \]
		When $t=0$, $\phi_X(0)= 1$ for any chf, and hence we obtain that
		\vspace{-2mm}
		\[ \phi_X(t) = e^{-|t|} \quad\quad t \in {\cal R}  \]
	\end{block}
}
\frame{\vspace{-2mm}
\begin{block}{Theorem C1.}
If a random variable $X$ has finite $E|X|^r$ for a positive integer $r$, then \vspace{-2mm}
\[ \frac{d^r \phi_X(t)}{dt^r}\bigg|_{t=0} = \dot{\imath}^r E(X^r)  \]
\end{block}
\vspace{-2mm}
\begin{block}{}
	\begin{itemize}
		\item
		The proof will be given in the next lecture.
		\item
	Thus, like the mgf, the chf can be used to calculate the moments, but we have to first know the existence of moments.
	\item
	For the Cauchy distribution, the chf is $e^{-|t|}$. 
	
Note that this chf is not differentiable at $t=0$ and previously we showed that the expectation of $X$ does not exist.		
	\end{itemize}
	\end{block}
		\vspace{-2mm}
		\begin{alertblock}{Uniqueness and inversion}
			Similar to the mgf, the chf characterizes the distribution in the sense that there is a one-to-one correspondence between chf and cdf.
			\vspace{1mm}
			
			We state some inversion formulas.
			\vspace{1mm}
			
			The proof of the following theorem  is omitted. 
		\end{alertblock}}
		\frame{
		\vspace{-2mm}
		\begin{alertblock}{Theorem C2}
			Let $X$ be a random variable with cdf $F$  and chf $\phi$.
			
			For any real numbers $y<u$, \vspace{-2mm}
			\[  P(y<X<u) + \frac{P(X=y)+P(X=u)}{2}
			= \lim_{T \rightarrow \infty} \int_{-T}^T \frac{e^{-\dot{\imath} ty} -e^{-\dot{\imath} tu} }{2\pi \dot{\imath} t} \phi(t) dt \]
		\end{alertblock}
		\vspace{-2mm}
		\begin{block}{Theorem C3.}
			Let $F$ be a cdf with chf $\phi$.
			\begin{itemize}
				\item[(i)]
				If $F$ is continuous at $y$ and $u$ with $y <u$, then 
			\[ F(u) -F(y) = \lim_{T \rightarrow \infty}
			\int_{-T}^T \frac{e^{-\dot{\imath} ty} -e^{-\dot{\imath} tu} }{2\pi \dot{\imath} t} \phi(t) dt \vspace{-2mm} \]
					\item[(ii)]
					If $F$ is continuous at $x$, then \vspace{-2mm}
					\[ F(u)  = \lim_{y \rightarrow -\infty} \lim_{T \rightarrow \infty}
					\int_{-T}^T \frac{e^{-\dot{\imath} ty} -e^{-\dot{\imath} tu} }{2\pi \dot{\imath} t} \phi(t) dt \vspace{-2mm} \]
					\item[(iii)]
					There is a one-to-one correspondence between cdf and chf.
				\end{itemize}
			\end{block}}
			\frame{
			\vspace{-2mm}
			\begin{block}{Proof.}
				(i) follows from Theorem 2C.
				
				(ii)-(iii) follow from the fact that any cdf can only have countably many discontinuous points and, hence,
				it is determined by the $F$ values at continuity points of $F$.
			\end{block}\vspace{-2mm}
			\begin{alertblock}{}
				If a cdf is continuous, then Theorem 3C(ii) gives an inversion formula.
				\vspace{1mm}
				
				For a discrete cdf, we have the following result whose proof is similar to that of Theorem 2C and is omitted.
			\end{alertblock}
\vspace{-2mm}
				\begin{alertblock}{Theorem C4.}
					Let $F$ be a discrete cdf with chf $\phi$.
					\begin{itemize}
						\item[(i)]
						If $F$ has a jump at $x$, then \vspace{-3mm}
						\[ F(x) - F(x-) = \lim_{T\rightarrow \infty} \frac{1}{2T} \int_{-T}^T e^{-\dot{\imath}tx} \phi(t) dt \vspace{-2mm} \]
						\item[(ii)]
						If $D$ is the set of all discontinuous points of $F$, then \vspace{-3mm}
						\[  \sum_{x \in D} [F(x)-F(x-)] = \lim_{T\rightarrow \infty} \frac{1}{2T} \int_{-T}^T |\phi(t)|^2 dt   \]
					\end{itemize}
				\end{alertblock}}
				\frame{
					\vspace{-2mm}
				\begin{alertblock}{}
					Theorem 4C(ii) gives a sufficient and necessary condition for a cdf to be continuous.
					\vspace{1mm}
					
					We know that $e^{-|t|}$ is a chf, and
					\vspace{-2.5mm}
					\[ \int_{-T}^T (e^{-|t|} )^2 dt \leq \int_{-\infty }^\infty e^{-2|t|} dt = 2 \int_0^\infty e^{-2t} dt =1 \quad\quad T>0 \vspace{-2.5mm} \]
					Hence, the corresponding cdf must be continuous.
					\vspace{1mm}
					
					In fact, in this case we know that the cdf corresponding to $e^{-|t|}$ has a pdf.
				\end{alertblock}	\vspace{-2mm}
				\begin{block}{Theorem C5.}
					Suppose that $X_1,X_2,...$ is a sequence of random variables with chf's $\phi_{X_n}(t)$, and 
					$ \lim_{n\rightarrow \infty} \phi_{X_n}(t) = \phi (t)$, $t \in {\cal R}$, 
					where  $\phi (t) $ is continuous in $t$.
					
					Then $\phi$ must be the chf of  a random variable $X$ and 
					for all $x$ at which $F_X(x)$ is continuous, \vspace{-1mm}
					\[ \lim_{n\rightarrow \infty} F_{X_n}(x) = F_X(x)\]
				\end{block}\vspace{-2mm}
				\begin{block}{}
					The proof is omitted. 
				\end{block}
				}
\frame{\vspace{-2mm}
							\begin{alertblock}{Symmetric random variables}
							A random variable $X$ is symmetric about 0 if $-  X$ has the same distribution as \vspace{1mm}
							$\! X$. 
						\end{alertblock}
						\vspace{-2mm}		
						\begin{alertblock}{Theorem C6.}
							A random variable  $X$ is symmetric about 0 iff
							its chf $\phi_X$ is real-valued function.
						\end{alertblock}\vspace{-2mm}
							\begin{alertblock}{Proof.}
								\begin{itemize}
									\item
									If $X$ and $-X$ have the same
									distribution, then
									$\phi_X(t)=\phi_{-X}(t)$.
									
									But $\phi_{-X}(t)=\phi_{X}(-t)$.
									
									Then
									$\phi_X(t)=\phi_{X}(-t)$.
									
									Note that $\sin (-t X) =
									- \sin(t X)$ and $\cos (tX) = \cos (-t X)$.
									
									Hence $E[\sin (t X) ]= 0$ and, thus, $\phi_X$ is real-valued.
									\item
									If $\phi_X$ is real-valued, then
									$\phi_X (t) = E[\cos (t X)]$ and $\phi_{-X}(t)
									=\phi_X(-t)=\phi_X(t)$.
									
									By Theorem C3, $X$ and $-X$ must have the same distribution.
								\end{itemize}
							\end{alertblock}
						}
\frame {
	\begin{alertblock}{Inequalities}
		Inequalities are useful for statistical theory.
	\end{alertblock}\vspace{-2mm}
	\begin{block}{Theorem 3.6.1 (Chebychev's inequality)}
		Let $X$ be a random variable and let $g(x)$ be  a nonnegative function. For any $r >0$, \vspace{-2mm}
		\[ P(g(X) \geq r ) \leq \frac{E[g(X)] }{r} \]
	\end{block}\vspace{-2mm}
	\begin{block}{Proof.}
		Assuming that $X$ has a pdf (if $X$ has a pmf, we replace integral by summation), we have  \vspace{-2mm}
		\beqrs
		E[ g(X) ] & = & \int_{-\infty}^\infty g(x)f_X(x) dx \geq
		\int_{-\infty}^\infty I( \{ g(x) \geq r\} )  g(x)f_X(x) dx \\
		& \geq & r \int_{-\infty}^\infty I( \{ g(x) \geq r\} )  f_X(x) dx
		= r \int_{ \{ g(x) \geq r\}} f_X(x) dx \\
		& = & r P(g(X) \geq r )
		\eeqrs
		where $I(A) $ is the indicator function of the set $A$.
	\end{block}
}
\frame{
	\vspace{-2mm}
	\begin{alertblock}{Different forms of  Chebychev's inequality}
		\begin{itemize}
			\item
			If $g$ is nondecreasing, then another form of  Chebychev's inequality is, for $\epsilon >0$, \vspace{-2mm}
			\[ P(X \geq \epsilon ) \leq \frac{E[g(X)] }{g(\epsilon )} \vspace{-2mm} \]
			\item
			Suppose that $X$ has expectation $\mu$ and variance $\sigma^2$.
			
			For $g(x) = (x- \mu)^2/\sigma^2$, we have \vspace{-2mm}
			\[ P(|X-\mu| \geq t\sigma ) = P\left( \frac{(X-\mu)^2}{\sigma^2} \geq t^2 \right) \leq \frac{1}{t^2} E\frac{(X-\mu)^2}{\sigma^2} = \frac{1}{t^2}\vspace{-2mm}  \]
			\item
			If $X$ has a finite $k$th moment with an integer $k$, then, for $t>0$,
			\vspace{-2mm}
			\[ P(|X-\mu| \geq t )  \leq \frac{E|X-\mu|^k}{t^k} \vspace{-2mm} \]
			\item
			If $X$ has a finite mgf $M_X(t)$ for $t \in (-h,h)$, then, for $r>0$ and $t>0$,  \vspace{-2mm}
			\[ P(X \geq r )  \leq \frac{E(e^{tX})}{e^{tr}} = \frac{M_X(t)}{e^{tr}}, \quad \quad
			P(X \leq -r )  \leq \frac{E(e^{-tX})}{e^{tr}} = \frac{M_X(-t)}{e^{tr}}\vspace{-2mm}  \]
			\[ P(|X| \geq r) \leq \frac{M_X(t)+M_X(-t)}{e^{tr}} \]
		\end{itemize}
	\end{alertblock}
}
\frame{ \vspace{-1mm}
	\begin{block}{}
		Chebychev's inequality is useful, but sometimes it is too loose because it does not require much from the distribution of $X$ except some moment conditions.
		\vspace{1mm}
		
		More useful probability inequality can be derived when we know something about the distribution of $X$.
	\end{block}\vspace{-2mm}
	\begin{alertblock}{Cauchy-Schwartz's inequality}
		This is another simple but very useful inequality.
		\vspace{1mm}
		
		If $X$ and $Y$ are random variables with $ E(X^2) < \infty$ and $ E(Y^2) < \infty$, then
		the following Cauchy-Schwartz's inequality holds: \vspace{-1mm}
		\[ [E(XY)]^2 \leq E(X^2)E(Y^2) \vspace{-1mm} \]
		with equality holds iff $P(X = cY ) =1$ for a constant $c$.
		\vspace{1mm}
		
		In fact, we also have \vspace{-1mm}
		\[ [E|XY| ]^2 \leq E(X^2)E(Y^2)  \]
	\end{alertblock}
}
\frame{
	\begin{alertblock}{Proof of Cauchy-Schwartz's inequality}
		Let $a^2 = E(X^2) < \infty$ and $b^2 = E(Y^2) < \infty$.
		\vspace{1mm} 
		
		For any $t >0$ and $s >0$, \vspace{-2mm}
		\[ 2\sqrt{st} \leq s + t \quad \quad \mbox{because} \ \ (\sqrt{s} - \sqrt{t})^2 \geq 0 \vspace{-2mm}\]
		Letting $s = X^2/a^2$ and $t = Y^2/b^2$, we obtain \vspace{-2mm}
		\[ \frac{|X Y| }{ab} \leq \frac{X^2}{2a^2} +  \frac{Y^2}{2b^2} \quad \quad \mbox{hence} \quad \quad
		\frac{E |X Y| }{ab} \leq \frac{E(X^2)}{2a^2} +  \frac{E(Y^2)}{2b^2} = 1 \vspace{-2mm} \]
		which means \vspace{-1mm}
		\[ [E|XY|]^2 \leq a^2b^2 = E(X^2)E(Y^2) \vspace{-1mm} \]
		The other inequality follows since \vspace{-2mm}
		\[ [E(XY)]^2 \leq [E|XY|]^2 \leq  E(X^2)E(Y^2) \vspace{-2mm} \]
		We next consider what happens if the equality holds.
		
		If  $[E|XY|]^2  = E(X^2)E(Y^2) $, then \vspace{-2mm}
		\[ \frac{E |X Y| }{ab} = \frac{E(X^2)}{2a^2} +  \frac{E(Y^2)}{2b^2} \quad \mbox{i.e.,} \quad
		E\left( \frac{|X|}{a} - \frac{|Y|}{b} \right)^2 = 0 \]
	\end{alertblock}}
	\frame{
		\begin{alertblock}{}
			It follows from a result established previously that \vspace{-2mm}
			\[  P\left( \frac{|X|}{a} - \frac{|Y|}{b} =0 \right) =1  \quad \mbox{i.e.,} \quad
			P \left( |X| = \frac{a}{b} |Y| \right) = 1 \vspace{-2mm} \]
			Finally, consider the situation where $[E(XY)]^2 = E(X^2)E(Y^2)$.
			
			Since $[E(XY)]^2 \leq [E|XY|]^2$, this implies \vspace{-2mm}
			\[ [E(XY)]^2 = [E|XY|]^2 = E(X^2)E(Y^2) \vspace{-2mm} \]
			and, by the early proof,
			$P ( |X| = \frac{a}{b} |Y| ) = 1$.
			
			From  $[E(XY)]^2 = [E|XY| ]^2 $, we must have
			$\pm E(XY) = E|XY|$.
			
			Suppose $E(XY) = E|XY|$ (the proof for $-E(XY)= E|XY|$ is similar).
			
			Since $|XY|  - XY \geq 0$, $E| XY| - E(XY) =0$ implies
			$P ( |XY| = XY ) = 1$.
			
			Combining this with the early result, we must have
			$P ( X = \frac{a}{b} Y ) = 1$.
		\end{alertblock}\vspace{-2mm}
		\begin{block}{}
			Cauchy-Schwartz's inequality
			is a special case of the following result.
			\begin{itemize}
				\item
				H\"{o}lder's inequality:
				If $p$ and $q$ are positive constants satisfying $p>1$ and $p^{-1}+q^{-1} =1$ and $X$ and $Y$ are random variables, then \vspace{-2mm}
				\[ E|XY| \leq (E|X|^p)^{1/p} (E|Y|^q)^{1/q} \]
			\end{itemize}
		\end{block}
	}
	\frame{\vspace{-1mm}
		\begin{block}{}
			Using H\"{o}lder's inequality, we can obtain the following two inequalities.
			\begin{itemize}
				\item
				Liapounov's inequality: If $r$ and $s$ are constants satisfying $1 \leq r \leq s$ and $X$ is a random variable, then \vspace{-2mm}
				\[ (E|X|^r)^{1/r} \leq (E|X|^s)^{1/s} \vspace{-2mm}\]
				\item
				Minkowski's inequality: If $p \geq 1$ is a constant and $X$ and $Y$ are random variables, then \vspace{-2mm}
				\[ (E|X+Y|^p)^{1/p} \leq (E|X|^p)^{1/p} + (E|Y|^p)^{1/p} \]
			\end{itemize}
		\end{block}\vspace{-2mm}
		\begin{alertblock}{Convex set and function}
			\begin{itemize}
				\item
				A set $A \subset {\cal R}^k $ is convex iff $x \in A$, $y \in A$, and $t \in (0,1)$ imply \vspace{-2mm}
				\[ tx + (1-t) y \in A \vspace{-2mm} \]
				\item
				A function  $g$ from a convex $A \subset {\cal R}^k$ to ${\cal R}$ is convex iff $x \in A$, $y \in A$, and $t \in (0,1)$ imply \vspace{-2mm}
				\[ g(tx+(1-t)y) \leq t \, g(x) + (1-t)g(y) \vspace{-2mm} \]
				and $g$ is strictly convex iff the previous inequality holds with $\leq $ replaced by $<$.
			\end{itemize}
		\end{alertblock}
		
	}
	\frame{
		\begin{alertblock}{}
			\begin{itemize}
				\item
				If $g$ is twice differentiable on a convex $A$, then a necessary and sufficient condition for $g$ to be convex (or strictly convex) is that the $k \times k$ second order partial derivative matrix $\partial^2 g / \partial x \partial x' $ is nonnegative definite (or positive definite).
				\item
				If $g$ is convex, then $-g$ is concave.
				\item
				The following is a very useful inequality in statistics.
			\end{itemize}
			\vspace{-10cm}
			
			\begin{figure}[h]
				\begin{center}
					%\caption{Graph of $y=\sin^2(x)$ in Example 2.1.2.}
					\includegraphics[height=8in, width=5in]{figure406.pdf}
				\end{center}
			\end{figure}
			
		\end{alertblock}
	}
	\frame{
		\begin{block}{Jensen's inequality}
			If $g$ is a convex function on a convex $A \subset {\cal R}$ and $X$ is a random variable with $P(X \in A)=1$, then \vspace{-2mm}
			\[ g(E(X)) \leq E[g(X)] \vspace{-1mm} \]
			provided that the expectations exist. If $g$ is strictly convex, then $\leq $ in the previous inequality can be replaced by $<$ unless
			$P(g(X) = c)=1$ for a constant $c$.
		\end{block}\vspace{-2mm}
		\begin{block}{}
			Jensen's inequality also holds for a convex $g$ defined on $A \subset {\cal R}^k$ and a random vector $X$ defined on ${\cal R}^k$ introduced in Chapter 4.
		\end{block}\vspace{-2mm}
		\begin{alertblock}{Proof of Jensen's inequality}
			Let $l(x) = a+ b x$ be the tangent line to $g(x)$ at $g(E(X))$ (see the figure).
			
			Since $g$ is convex, $g(x) \geq a+bx$ for all $x$ and, hence,
			\vspace{-2mm}
			\[ E[g(X)] \geq E(a+bX) = a+bE(X) = l(E(X)) = g(E(X)). \vspace{-2mm} \]
			If  $E[g(X)]=g(E(X))$, then $P(g(X)=a+bX)=1$, which cannot occur if
			$g$ is strictly convex and $g(X)$ is not a constant.
		\end{alertblock}}
		\frame{
			\begin{block}{Examples}
				\begin{itemize}
					\item
					The function $g(x) =  x^{-1}$ is strictly convex.
					Hence, \vspace{-2mm}
					\[ (EX)^{-1} < E(X^{-1}) \vspace{-2mm} \]
					unless $P(X=c)=1$ for a constant $c$.
					\item
					The function $g(x) = - \log x$ is strictly convex ($\log x$ is strictly concave). Then\vspace{-2mm}
					\[ - \log (EX) < - E( \log X ) \quad \quad \mbox{i.e.,} \quad \quad E(\log X) < \log (EX)  \vspace{-2mm} \]
					unless $P(X=c)=1$ for a constant $c$.
					\item
					Let $f$ and $g$ be positive functions satisfying $0 < \int_{-\infty}^\infty g(x) dx \leq \int_{-\infty}^\infty f(x) dx =1$.
					We want to show that \vspace{-2mm}
					\[ \int_{-\infty}^\infty f (x) \log  \frac{g(x)}{f(x)} dx \leq 0 \vspace{-2mm} \]
					Note that $f$ is a pdf.
					
					Let $X \sim f$ and $Y =  \frac{g(X)}{f(X)}$.
				\end{itemize}
			\end{block}}
			\frame{
				\begin{block}{}
					\ By Jensen's inequality, \vspace{-2mm}
					\beqrs
					\int_{-\infty}^\infty f (x) \log  \frac{g(x)}{f(x)} dx
					& = &  E\left(\log \frac{g(X)}{f(X)}\right) = E ( \log Y) \\
					& \leq & \log (EY)  =   \log \left( E \frac{g(X)}{f(X)} \right) \\
					& = & \log \left( \int_{-\infty}^\infty \frac{g(x)}{f(x)} f(x) dx \right)  =  \log \left( \int_{-\infty}^\infty g(x) dx \right) \\
					& \leq & \log \left( \int_{-\infty}^\infty f(x) dx \right)  =  \log(1) \\
					& = & 0
					\eeqrs
					where the last inequality follows from the fact that $\log x$ is increasing.
					\vspace{1mm}
					
					Also, the strict $<$ holds unless $P(f(X)= g(X))=1$.
				\end{block}
			}
				
\end{document}
