% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.

% Make two column format for LaTex 2e.


\documentclass[11pt,twocolumn]{article} %
\usepackage[dvips]{graphicx,graphics,color}
\usepackage{times,amsmath,amsfonts}

\setlength{\textheight}{10in} \setlength{\textwidth}{6.875in}
\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{-0.5in}
\setlength{\headheight}{0in} \setlength{\headsep}{0in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.2in}  % Centers text.
\setlength{\evensidemargin}{-.2in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-8pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-3pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand{\bq}{\begin{eqnarray*}}
\newcommand{\eq}{\end{eqnarray*}}
\begin{document}

% Don't want date printed
\date{March 15, 2005}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 324: Lecture 14\\
Point Estimation}

% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}

% For two authors (default example)
\maketitle \thispagestyle{empty}

\section*{Concepts}
\begin{enumerate}


\item {\em Population} of interest is a collection of measurable objects we are
studying. Let $X_1, \cdots, X_n$ be a random sample from the
population. A {\em point estimate} of population parameter $\theta$
is a number that reasonably approximate $\theta$. It is obtained by
computing a suitable statistic from the given sample data. A {\em
point estimator} of $\theta$ is a statistic for getting the point
estimate.\\

{\em Example.} Let $X_i$ be a Bernoulli random variable with
$P(X_i=1)=p$. %Then $X =\sum_{i=1}^n X_i$ is a Binomial random
%variable with $n$ and $p$.
We can estimate $p$ using the estimator $\hat p = \bar X$.
%parameters.
%Note that a Binomial random variable with $n=1$ is a
%Bernoulli random variable. The following example simulates 1000 coin
%tosses.
%
%\begin{verbatim}
%> a<-rbinom(1000,1,0.5)
%> a
%   [1] 0 0 1 0 0 1 0 1 1 0 0 ...
%> sum(a)/1000
%[1] 0.517
%> b<-rbinom(1000000,1,0.5)
%> sum(b)/1000000
%[1] 0.500545
%\end{verbatim}

\item
$$\theta = \hat \theta + \mbox{error of estimation.}$$
We define the {\em bias} of estimator $\hat \theta$ to be
$$Bias(\hat \theta) = \mathbb{E}(\hat \theta - \theta) = \mathbb{E}(\hat \theta) - \theta.$$ It
measures the biasness (estimation error) of estimator $\hat \theta$
in an average sense.


\item  Then sample mean $\bar X$ and sample variance $S^2$ are
unbiased estimators of population mean $\mu$ and population variance
$\sigma^2$
respectively.\\
{\em Proof.} Note that
$$S^2 = \frac{1}{n-1}\sum_{i=1} (X_i -\bar X)^2= \frac{1}{n-1}\Big[ \sum_{i=1}^n X_i^2 - n \bar X^2 \Big].$$
Then using the fact $\mathbb{E} (\bar X)^2 = \mathbb{V} \bar X +
(\mathbb{E} \bar X)^2 = \sigma^2/n + \mu^2$, it can be shown that
$\mathbb{E} S^2 = \sigma^2$.


\item There may be many unbiased estimators of $\theta$. Given two unbiased
estimators $\hat \theta_1$ and $\hat \theta_2$ of $\theta$. We
choose one that gives less variance. If $\mathbb{V}(\hat \theta_1)
\leq \mathbb{V}(\hat \theta_2)$, $\hat \theta_1$ is called more {\em
efficient} than $\hat \theta_2$. An efficient estimator has less
variability so we are more likely to make an estimate close to the
true parameter value.
%The following coin flipping example clearly
%demonstrate this.
%
%\begin{verbatim}
%> a<-rbinom(1000,1,0.5)
%> a
%   [1] 0 0 1 0 0 1 0 1 1 0 0 ...
%> mean(a)
%[1] 0.517
%> mean(a[1:3])
%[1] 0.3333333
%> mean(a[1:9])
%[1] 0.4444444
%> mean(a[1:11])
%[1] 0.3636364

\end{verbatim}



\item Among all unbiased estimators, we choose the most efficient estimator called
the {\em minimum variance unbiased estimator} (MVUE). The MVUE is an
unbiased estimator with the smallest variance. MVUE is the most
efficient estimator. An efficient estimator $\hat \theta$ will
produce an estimate closer to the true parameter $\theta$. $\bar X$
is MVUE for $\mu$ (we will not prove this statement).


\end{enumerate}

\section*{Problems}

Given a random sample $X_1,\cdots, X_n$, a linear estimator of
parameter $\theta$ is an estimator of form $$\hat \theta =
\sum_{i=1}^n c_i X_i.$$ Then it can be shown that $\bar X$ is the
MVUE for population mean $\mu$ among all possible linear
estimators.\\ {\em Proof.} Case $n=2$ will be proved. The general
statement follows inductively. Consider linear estimators $$\hat \mu
= c_1 X_1 + c_2 X_2.$$ To be unbiased, $c_1 + c_2 =1$. To be most
efficient among all unbiased linear estimators, the variance has to
be minimized. The variance is
$$\mathbb{V} \hat \mu = c_1^2 \mathbb{V} X_1 + c_2^2\mathbb{V} X_2= \big[c_1^2 + (1-c_1)^2\big]\sigma^2$$
The quadratic term in the bracket $2c_1^2 -2c_1 + 1$  is minimized
when $c_1=1/2$.\\


{\bf Assigned Problems:} Exercise 6.10, Exercise 6.16. {\bf Homework
4} due on March 17th. 9:30AM. Solve assigned problems in Lecture
11-14. Please read Chapter 6.
\end{document}