% $Header: /cvsroot/latex-beamer/latex-beamer/examples/beamerexample5.tex,v 1.22 2004/10/08 14:02:33 tantau Exp $

\documentclass[11pt]{beamer}
\usepackage{amscd}

%\usetheme{Madrid} % Darmstadt, Boadilla


\mode<presentation> {
  \usetheme{Madrid}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsbsy,alltt,amsmath,times}
\usepackage{epsfig, color, graphicx, rotate, colordvi}
%\usepackage{fancybox,fancyhdr,times,subfigure}
\usepackage{hyperref, mathptmx}


\setbeamercovered{dynamic}

\newcommand{\Lang}[1]{\operatorname{\text{\textsc{#1}}}}

\newcommand{\Class}[1]{\operatorname{\mathchoice
  {\text{\sf \small #1}}
  {\text{\sf \small #1}}
  {\text{\sf #1}}
  {\text{\sf #1}}}}

\newcommand{\NumSAT}      {\text{\small\#SAT}}
\newcommand{\NumA}        {\#_{\!A}}

\newcommand{\barA}        {\,\bar{\!A}}

\newcommand{\Nat}{\mathbb{N}}
\newcommand{\Set}[1]{\{#1\}}

\newtheorem{thm}{Result}%[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{defn}[thm]{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{rem}{Remark}

\newcommand{\cred}{\textcolor {red}}
\newcommand{\cblue}{\textcolor {blue}}
\newcommand{\cgreen}{\textcolor {green}}

\def\ul{\underline}

\def\sign{\;{\rm sign}}
\def\rank{\;{\rm rank}}
\def\disp{\displaystyle}
\def\sn{\sqrt{n}}
\def\Pr{\mathrm{P}}

\def\Cov{\;{\rm Cov}}
\def\L{\underline L}
\def\LL{\overline L}

\def\balpha{{\boldsymbol{\alpha}}}
\def\cal{\mathcal}
\def\conas{\stackrel{a.s.} {\longrightarrow}}         % conv. a.s.
\def\conp{\stackrel{\cal P} {\longrightarrow}}        % conv. in probability
\def\conD{\stackrel{\mathcal{L}} {\longrightarrow}}        % conv. in distribution
\def\conw{\stackrel{\mathcal{L}} {\rightarrow}}        % conv. in distribution
\def\hat{\widehat}
%\def\endproof{\begin{flushright}$\lozenge$\end{flushright}}
\def\Proof{\noindent{\it Proof }}
\def\endproof{\hfill{$\Box$}}
\newcommand{\wcon}{\stackrel{\mathcal{L}}{\longrightarrow}}
\def\Var{\mbox{var}}
\def\br{\begin{array}}
\def\er{\end{array}}
\def\ep{\epsilon}
%%%%%%%%%%%%%%% MY DEFINITIONS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Put your own definitions
%
\def\VaR{{\rm VaR}}
\def\E{{\rm E}}
\def\beqrs{\begin{eqnarray*}}           %\beqrs
\def\eeqrs{\end{eqnarray*}}             %\eeqrs

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\pgfdeclaremask{tu}{beamer-tu-logo-mask}
\pgfdeclaremask{computer}{beamer-computer-mask}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerimage}{beamer-computer}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerworkingimage}{beamer-computerred}
\pgfdeclareimage[mask=tu,height=.5cm]{logo}{beamer-tu-logo}

\logo{\pgfuseimage{logo}}

\title[Stat 609 Lecture 22]{Stat 609: Mathematical Statistics \\ Lecture 22}
\author[UW-Madison]{Jun Shao}
\institute[Statistics]{%
%  Department of Statistics\\
%  University of Wisconsin\\
%  Madison, WI 53706, USA\\
%~~~\\
  }
\date[2015]{}

\colorlet{redshaded}{red!25!bg}
\colorlet{shaded}{black!25!bg}
\colorlet{shadedshaded}{black!10!bg}
\colorlet{blackshaded}{black!40!bg}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\def\radius{0.96cm}
\def\innerradius{0.85cm}

\def\softness{0.4}
\definecolor{softred}{rgb}{1,\softness,\softness}
\definecolor{softgreen}{rgb}{\softness,1,\softness}
\definecolor{softblue}{rgb}{\softness,\softness,1}

\definecolor{softrg}{rgb}{1,1,\softness}
\definecolor{softrb}{rgb}{1,\softness,1}
\definecolor{softgb}{rgb}{\softness,1,1}

\newcommand{\Bandshaded}[2]{
  \color{shadedshaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\Band}[2]{
  \color{shaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\BaenderNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{input tapes}{}}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{Zwei Eingabebänder}{}}
  \pgfputat{\pgfxy(0,4.25)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \ignorespaces}

\newcommand{\BaenderHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{input tapes}{}}
  \color{shaded}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Bandshaded{}{}}

  \color{blackshaded}
  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{Zwei Eingabebänder}{}}%
  \color{blackshaded}
  \pgfputat{\pgfxy(0,4.25)}{\Bandshaded{}{}}
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}%

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \ignorespaces}

\newcommand{\Slot}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.0)}%
    \pgfbezier{\pgfxy(0.1,3.95)}{\pgfxy(0.05,3.9)}{\pgfxy(0,3.9)}%
    \pgfbezier{\pgfxy(-0.05,3.9)}{\pgfxy(-0.1,3.95)}{\pgfxy(-0.1,4.0)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\SlotZwei}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.25)}%
    \pgfbezier{\pgfxy(0.1,4.25)}{\pgfxy(0.05,4.15)}{\pgfxy(0,4.15)}%
    \pgfbezier{\pgfxy(-0.05,4.15)}{\pgfxy(-0.1,4.2)}{\pgfxy(-0.1,4.25)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\ClipSlot}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4cm}}}{\pgfxy(0.2,1.5)}\ignorespaces}

\newcommand{\ClipSlotZwei}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4.25cm}}}{\pgfxy(0.2,1.25)}\ignorespaces}


\AtBeginSection[]{\frame{\frametitle{Outline}\tableofcontents[current]}}

\begin{document}

%\frame{\titlepage}

\frame {\vspace{-2mm}
\frametitle{Chapter 6. Principles of Data Reduction\\
Lecture 22: Sufficiency}
\vspace{-1mm}
    \begin{alertblock}{Data reduction}
We consider a sample $X= (X_1,...,X_n)$, $n >1$, from a population of interest
(each $X_i$ may be a vector and $X$ may not be a random sample, although most of the time we consider a random sample).
\vspace{1.5mm}

Assume the population is indexed by $\theta$, an unknown parameter vector.
\vspace{1.5mm}

Let ${\cal X}$ be the range of $X$
\vspace{2mm}

Let $x$ be an observed data set,  a realization of $X $.
\begin{itemize}
\item
We want to use the information about $\theta$ contained in $x$.
\item
The whole $x$ may be hard to interpret, and hence we summarize the information by
 using a few key features (statistics).

For example, the sample mean, sample variance, the largest and smallest order statistics.
\item
Let $T(X )$ be a statistic. For $T$, if $x \neq y$ but $T(x )=T(y )$,
then $x$ and $y$ provides the same information and can be treated as the same.
\end{itemize}
\end{alertblock}}
\frame{
\begin{alertblock}{}
\begin{itemize}
\item
$T$ partitions ${\cal X}$ into sets \vspace{-2mm}
\[ A_t = \{ x : T(x ) = t \}, \quad t \in
{\cal T} \mbox{ (the range of $T$)} \vspace{-2mm} \]
All points in $A_t$ are treated the same if we are interested in $T$ only.
\item
Thus, $T$ provides a data reduction.
\item
We wish to reduce data as much as we can, but not lose any information
about $\theta$ (or at least important information).
\end{itemize}
\end{alertblock}\vspace{-2mm}
\begin{block}{Sufficiency}
A sufficient statistic for $\theta$ is a statistic that captures all the information
about $\theta$ contained in the sample.

Formally we have the following definition.
\end{block}\vspace{-2mm}
\begin{alertblock}{Definition 6.2.1 (sufficiency)}
A statistic $T(X )$ is sufficient
for $\theta$ if the conditional distribution of
$X$ given $T(X ) = T(x )$ does not depend on $\theta$.
\end{alertblock}\vspace{-2mm}
\begin{alertblock}{}
\begin{itemize}
\item
The sufficiency depends on the parameter of interest.
\end{itemize}
\end{alertblock}}
\frame{
\begin{alertblock}{}
\begin{itemize}
\item
If $X$ is discrete, then so is $T(X )$ and sufficiency means that
$P(X = x |T(X ) = T(x ))$ is known, i.e., it does not depend on any unknown quantity.
\item
Once we observe $x$ and
compute a sufficient
statistic $T(x )$, the original data $x$ do not contain any further
information concerning $\theta$ and can be discarded, i.e., $T(x)$ is all we need regarding $\theta$.
\item
If we do need $x$, we can simulate a sample $y$ from
$P(X = y |T(X ) = T(x ))$ since it is known; the observed $y$ may not be the same as $x$, but $T(x )=T(y )$.
\end{itemize}
     \end{alertblock}\vspace{-2mm}
         \begin{block}{Example 6.2.3 (binomial sufficient statistic)}
Suppose that  $X_1,...,X_n$ are iid Bernoullie
variables with probability $\theta $.

The joint pmf is\vspace{-2mm}
\[ f_\theta (x_1,...,x_n) = \left\{ \br{ll}
{\displaystyle \prod_{i=1}^n}\theta^{x_i} (1-\theta)^{1-x_i}
& \quad x_i=0,1, i=1,...,n\vspace{2mm} \\
0 & \quad \mbox{otherwise} \er \right. \vspace{-2mm} \]
Consider the statistic $T(X)=\sum_{i=1}^n X_i$, which is the number of
ones in $X$.
\end{block}}
\frame{
\begin{block}{}
To show $T$ is sufficient for $\theta$, we compute the conditional probability
$P(X =x |T=t)$.

For $t=0,1,...,n$, let \vspace{-2mm}
\[  B_t =\left\{ x=(x_1,...,x_n) : \ x_i =0,1, \ \sum_{i=1}^n x_i =t \right\}. \vspace{-2mm} \]
If $x \not \in B_t$, then $P(X=x | T=t )=0$.

If $x \in B_t$, then \vspace{-2mm}
\[ P(X=x,T=t) = P(X = x ) = f_\theta (x )
= \theta^t (1-\theta )^{n-t} . \vspace{-2mm}\]
Also, since $T \sim binomial (n,p)$, \vspace{-2mm}
\[ P(T=t) = {n \choose t} \theta^t (1-\theta )^{n-t} \vspace{-2mm} \]
Then, for $t=0,1,...,n$, \vspace{-2mm}
\[ P( X=x | T=t ) = \frac{ P( X=x , T=t )}{P(T=t)}
= \frac{1}{{n \choose t}} \ \ x \in B_t \vspace{-2mm} \]
is a known pmf (does not depend on $\theta$).
\end{block}}
\frame{
\begin{block}{}
Hence $T(X)$ is sufficient for $\theta$.
\vspace{1mm}

For any realization $x$ of $X$, $x$ is a sequence of $n$ ones and zeros.
\vspace{1mm}

Since $\theta$ is the probability of a one and
$T$ is the frequency of ones in $x$, it has all the information about $\theta$.
\vspace{1mm}

Given $T=t$, what is left in
the data set $x$ is the redundant information about the positions of $t$
ones, and we can reproduce the data set $x$ if we want by using $T=t$.
\end{block}\vspace{-2mm}
    \begin{alertblock}{How to find sufficient statistics?}
To verify that a statistic $T$ is a sufficient statistic for $\theta$ by definition, we must verify that for any fixed values of $x$,
the conditional distribution $X|T(X)=T(x)$ does not depend on $\theta$.
\vspace{2mm}

This may not be easy but at least we can try.
\vspace{2mm}

But how do we find the form of $T$?
By guessing a statistic $T$ that might be
sufficient and computing the conditional distribution of $X|T=t$?
\vspace{2mm}

For families of populations having pdfs or pmfs, a simple
way of finding sufficient statistics is to use the
following factorization theorem.
\end{alertblock}}
\frame{
\begin{block}{Theorem 6.2.6 (the Factorization Theorem)}
Let $f_\theta (x )$ be the joint pdf or pmf of the sample $X$.
A statistic $T(X)$ is sufficient for $\theta$ iff there are functions $h$
(which does not depend on $\theta$)
and $g_\theta $ (which depends on $\theta$) on the range of $T$ such that\vspace{-2mm}
\[ f_\theta (x ) = g_\theta \bigl(T( x) \bigr) h(x ) . \]
\end{block}\vspace{-2mm}
\begin{alertblock}{}
In the binomial example,
$f_\theta (x) =  g_\theta \bigl(T( x) \bigr) h(x )$
if we set \vspace{-2mm}
\[ g_\theta (t) = \theta^t (1-\theta )^{n-t} \ \ {\rm and} \ \
h(x ) = \left\{ \br{ll}
1 & \quad x_i=0,1, i=1,...,n\\
0 & \quad \mbox{otherwise} \er \right. \]
\end{alertblock}\vspace{-2mm}
\begin{block}{Proof of Theorem 6.2.6 for the discrete case.}
Suppose that $T(X)$ is sufficient.

Let $g_\theta (t) = P_\theta (T(X)=t)$ and
$h(x ) = P(X=x|T(X)=T(x))$.

Then \vspace{-3mm}
\beqrs
f_\theta (x) & = & P_\theta (X=x) = P_\theta (X=x, T(X) = T(x) ) \\
& = & P_\theta (T(X)=T(x)) P(X=x|T(X)=T(x)) \\
& = & g_\theta (T(x)) h(x)
\eeqrs
\end{block}}
\frame{ \vspace{-3mm}
\begin{block}{}
Suppose now that $f_\theta (x ) = g_\theta \bigl(T( x) \bigr) h(x )$ for $x \in {\cal X}$.

Let $q_\theta (t)$ be the pmf of $T(X)$ and $A_{x} = \{ y: T(y)=T(x) \}$.

Then, for any $x \in {\cal X}$, \vspace{-2mm}
\beqrs
\frac{f_\theta (x)}{q_\theta (T(x)) } & = & \frac{g_\theta (T(x)) h(x)}{q_\theta (T(x))}
=\frac{g_\theta (T(x)) h(x)}{P_\theta (T(X) = T(x)) } \\
& = & \frac{g_\theta (T(x)) h(x)}{\sum_{y \in A_x} f_\theta (y)}  = \frac{g_\theta (T(x)) h(x)}{\sum_{y \in A_x} g_\theta (T(y)) h(y)} \\
& = & \frac{g_\theta (T(x)) h(x)}{g_\theta (T(x))\sum_{y \in A_x}  h(y)} =  \frac{h(x)}{\sum_{y \in A_x}  h(y)}
\eeqrs
$~$ \vspace{-5mm}

which does not depend on $\theta$, i.e., $T$ is sufficient for $\theta$.
\end{block}\vspace{-2.5mm}
\begin{alertblock}{Example 6.2.4 (normal sufficient statistic)}
Let $X_1,...,X_n$ be iid $N(\mu,\sigma^2)$, $\theta = (\mu,\sigma^2)$; the joint pdf is \vspace{-3mm}
\beqrs
 f_\theta (x ) &=& \prod_{i=1}^n \frac{1}{\sqrt{2\pi}\sigma}
e^{-(x_i-\mu)^2/2\sigma^2}
 =  \frac{1}{(2\pi)^{n/2}\sigma^n} \exp \left( - \sum_{i=1}^n \frac{(x_i-\mu)^2}{2\sigma^2}
\right) \\
& = & \frac{1}{(2\pi)^{n/2}\sigma^n} \exp \left( -
\sum_{i=1}^n \frac{(x_i-\bar{x} )^2}{2\sigma^2} -\frac{n(\bar{x}-\mu)^2}{2\sigma^2} \right)
\eeqrs
\end{alertblock}}
\frame{
\begin{alertblock}{}
\[  =  \frac{1}{(2\pi)^{n/2}\sigma^n} \exp \left( -
\frac{(n-1)s^2}{2\sigma^2} -\frac{n(\bar{x}-\mu)^2}{2\sigma^2} \right)  \]
where $s^2 = (n-1)^{-1} \sum_{i=1}^n (x_i - \bar{x})^2$, the realization of the sample
variance $S^2 = (n-1)^{-1} \sum_{i=1}^n (X_i - \bar{X})^2$.
\vspace{1mm}

Hence, by Theorem 6.2.6, $(\bar{X},S^2)$ is a two-dimensional sufficient statistic for
$\theta = (\mu,\sigma^2)$.
\begin{itemize}
\item
If $\sigma^2$ is known, then $\bar{X}$ is sufficient for $\mu$.
\item
If $\mu$ is known, then $S^2$ is sufficient for $\sigma^2$.
\item
If both $\mu$ and $\sigma^2$ are unknown, we cannot say that $\bar{X}$ is sufficient for $\mu$
(or  $S^2$ is sufficient for $\sigma^2$); the correct statement is that
$\bar{X}$ and $S^2$ together is sufficient for $\mu$ and $\sigma^2$.
\item
We can also say that $(\bar{X},S^2)$ is sufficient for $\mu$ (or $\sigma^2$).
\end{itemize}
\end{alertblock}\vspace{-2mm}
\begin{block}{Sufficiency for a sub-family}
Let $\theta$ be a parameter and $\eta$ be a subset of components
of $\theta$.

If $T$ is sufficient for $\theta$, then it is also sufficient
for $\eta$.  \end{block}
}
\frame {
\begin{block}{Example 6.2.5 (sufficient order statistics)}
Let $X_1,...,X_n$ be iid with a pdf $f_\theta $ and $X_{(1)},...,X_{(n)}$ be the order statistics.

 The joint pdf of $X=(X_1,...,X_n)$ is \vspace{-2mm}
\[  \prod_{i=1}^n f_\theta (x_i) = \prod_{i=1}^n f_\theta (x_{(i)}) \vspace{-2mm}\]
where $x_{(1)},...,x_{(n)}$ are the ordered values of $x_1,...,x_n$.

Then, by the factorization theorem, $(X_{(1)},...,X_{(n)})$ is sufficient
for $\theta $.
\vspace{1mm}

Intuitively, given the order statistics, what is left in the original data set is the information regarding
the positions of $x_1,...,x_n$ and, hence, the set of order statistics is sufficient whenever positions of $x_i$'s
are not of interest.
\end{block}\vspace{-2mm}
\begin{alertblock}{One-to-one transformations of a sufficient statistic}
It follows from the factorization theorem that, if $T$ is sufficient and $U$ is a one-to-one function of $T$,
then $U$ is also sufficient. \vspace{2mm}

But this is also true in general by the definition of sufficiency.
\end{alertblock}
}
\frame{
\begin{block}{}
In the order statistics problem, $U=(U_1,...,U_n)$ is a one-to-one
function of $(X_{(1)},...,X_{(n)})$, where
$U_k = \sum_{i=1}^n X_i^k$, $k=1,...,n$.
\vspace{1mm}

Hence, $U$ is also sufficient for $\theta$.
\end{block}\vspace{-2mm}
    \begin{alertblock}{Example 6.2.8 (uniform sufficient statistic)}
Let $X_1,...,X_n$ be iid from $uniform(0,\theta )$, where $\theta >0$ is the unknown parameter.

The joint pdf of $X_1,...,X_n$ is \vspace{-2mm}
\[ \prod_{i=1}^n f_\theta (x_i)
= \prod_{i=1}^n \left[ \frac{1}{\theta} I( \{ 0<x_i<\theta \}) \right]
= \frac{1}{\theta^n} I( \{ 0 < x_{(n)} < \theta \}) \vspace{-2mm} \]
with $x_{(n)}$ being the largest value of $x_1,...,x_n$.
\vspace{1mm}

Thus, the largest order statistic $X_{(n)}$ is sufficient for $\theta$.
\vspace{1mm}

Intuitively, because $X_i \leq \theta$ for all $i$, if we observe $X_{(n)}$, then we know that $\theta \geq X_{(n)}$ and
the values of other $X_i$'s do not provide any additional information about $\theta$.
\vspace{1mm}

The same result holds when $X_1,...,X_n$ are iid from the discrete uniform distribution on $1,2,...,\theta$.
     \end{alertblock}
}

\frame { \vspace{-2mm}
    \begin{block}{Theorem 6.2.10 (exponential families)}
Let $X_1,...,X_n$ be iid from a pdf or pmf $f_\theta (x)$ that belongs to an exponential family: \vspace{-2mm}
\[ f_\theta (x) = h(x) c( \theta ) \exp \left( \sum_{j=1}^k w_j(\theta ) t_j(x) \right) \vspace{-2mm} \]
The joint pdf or pmf of $X=(X_1,...,X_n)$ is \vspace{-2mm}
\[ \prod_{i=1}^n f_\theta (x_i) =
\left[ \prod_{i=1}^n h(x_i) \right] [c(\theta )]^n
 \exp \left( \sum_{j=1}^k w_j(\theta ) \sum_{i=1}^n t_j(x_i) \right) \vspace{-2mm} \]
 It follows from the factorization theorem that the $k$-dimensional statistic \vspace{-2mm}
 \[ T(x) = \left(  \sum_{i=1}^n t_1(X_i),..., \sum_{i=1}^n t_k(X_i) \right) \vspace{-2mm} \]
 is sufficient for $\theta$.
     \end{block}}
     \frame{\vspace{-2mm}
    \begin{alertblock}{Sufficiency Principle}
    	Let $X$ be a sample from a population indexed by $\theta \in \Theta$.
    	
    	If $T(X )$ is sufficient for $\theta$, then
    	any inference about $\theta$ should depend on the sample only through the value $T(X )$.
    \end{alertblock}\vspace{-2mm}
\begin{alertblock}{}
	\begin{itemize}
		       \item
		       Another way to state the sufficiency principle is that, if $x$ and $y$ are two data points (realizations of $X$), then our decision or inference
		       about $\theta$ should be the same when $T(x)=T(y)$.
		\item
		The sufficiency principle says that in any inference procedure we should consider functions of a sufficient statistic only.
		\item
		In what sense we can be assured that using functions of a sufficient statistic is enough?
		\item
		First we should have a criterion to evaluate the performance of inference procedures.
		\item
		As an example, we consider here the problem of estimating a function $\vartheta = \psi (\theta )$, where $\psi$ is  a known function on the parameter space $\Theta$,
		but $\vartheta$ is unknown.
	\end{itemize}
\end{alertblock}
}

\frame{
	\begin{alertblock}{}
		\begin{itemize}
			\item
			Let $U(X)$ be a statistic used to estimate the unknown $\vartheta$. 
			
			A common criterion for the performance of $U(X)$ is
			the so-called mean squared error (mse) defined as \vspace{-2mm}
			\[ E_\theta [U(X)-\vartheta ]^2 = E_\theta [U(X) - \psi (\theta )]^2 , \quad \quad \theta \in \Theta \vspace{-2mm} \]
			where $E_\theta $ is the expectation with respect to the population indexed by $\theta$.
			\item
			We view $U(X) - \vartheta$ to be the estimation error, which is random since $X$ is random.
			The mse is simply the average of squared estimation error under the population indexed by $\theta$,
			and we want to choose a statistic such that the mse is as small as possible.
		\end{itemize}
	\end{alertblock}\vspace{-2mm}
	\begin{block}{Rao-Blackwell theorem}
		Let $X$ be a sample from a population indexed by $\theta \in \Theta$ and $T(X)$ be a sufficient
		statistic for $\theta$. If $U(X)$ is a statistic used to estimate $\vartheta = \psi (\theta )$ and $E_\theta [U(X)-\vartheta ]^2 < \infty$, then
		the statistic $h(T) = E[U(X)|T]$ satisfies
		\vspace{-2mm}
		\[ E_\theta [ h(T)- \vartheta ]^2 < E_\theta [U(X)-\vartheta ]^2 \quad \quad \mbox{$\theta \in \Theta $} \vspace{-2mm} \]
		unless $P_\theta (U(X) = h(T(X)))=1$, $\theta \in \Theta $.
	\end{block}
}
\frame{\vspace{-2mm}
	\begin{alertblock}{}
		\begin{itemize}
			\item
			The Rao-Blackwell theorem says that if $U(X)$ is not a function of the sufficient statistic $T$, then the new statistic $h(T) =
			E[U(X)|T]$ is better than $U(X)$ in terms of the mean squared error criterion.
			\item
			The theorem is meaningful if a $T$ other than the original data $X$ can be found (such as the minimal sufficient statistic).
			\item
			Because $E_\theta [U(X)-\vartheta ]^2 < \infty$, $E[U(X)|T]$ is well defined; in fact, we only need
			$E_\theta |U(X)| < \infty$ for every $\theta  \in \Theta$.
			\item
			Because $T$ is sufficient, $E[U(X)|T]$ does not depend on $\theta $ and is a statistic.
			\item
			The Rao-Blackwell theorem actually has a more general form considering a criterion other than the mean squared error.
		\end{itemize}
	\end{alertblock}
	\vspace{-2mm}
	\begin{block}{Proof.}
		For every $\theta \in \Theta$, \vspace{-2mm}
		\beqrs
		E_\theta [U(X)-\vartheta ]^2 &=& E_\theta \{ [U(X)-h(T)] +[ h(T) - \vartheta ] \}^2 \\
		& = & E_\theta [ U(X)-h(T)] ^2 + E_\theta [h(T)-\vartheta ]^2 \\
		&  & + \ \ 2 E_\theta [ U(X)-h(T)] [h(T)-\vartheta ] \\
		\eeqrs
	\end{block}}
	\frame{
		\begin{block}{}
			Using the properties of conditional expectations, we obtain that \vspace{-2mm}
			\beqrs
			E_\theta [ U(X)-h(T)] [h(T)-\vartheta ] & = &  E_\theta \big( E_\theta \big\{ [ U(X)-h(T)] [h(T)-\vartheta ] | T \big\} \big) \\
			& = & E_\theta \big( [h(T)-\vartheta ] E_\theta \big\{ [ U(X)-h(T)]  | T \big\} \big) \\
			& = & E_\theta \big( [h(T)-\vartheta ] E_\theta  [ U(X)|T] -h(T) \big) \\
			& = & 0
			\eeqrs
			Hence, \vspace{-2mm}
			\[   E_\theta [U(X)-\vartheta ]^2 =E_\theta [ U(X)-h(T)] ^2 + E_\theta [h(T)-\vartheta ]^2 >
			E_\theta [h(T)-\vartheta ]^2 \vspace{-2mm} \]
			unless $E_\theta [ U(X)-h(T)] ^2 = 0$, which implies $P_\theta (U(X)=h(T))=1$ from our previous discussion.
		\end{block}\vspace{-2mm}
		\begin{alertblock}{}
			The Rao-Blackwell theorem tells us that we should consider functions of a sufficient statistic (if one simpler than $X$ is available).
			\vspace{1mm}
			
			
			However, we still need to choose a function such that it provides the best procedure among all functions of the given sufficient statistic.
			\vspace{1mm}
			
			This will be treated in later chapters.
		\end{alertblock}}

\end{document}
