% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.

% Make two column format for LaTex 2e.
\documentclass[12pt,twocolumn]{article} %
\usepackage[dvips]{graphicx,graphics,color}
\usepackage{times,amsmath,amsfonts}

\setlength{\textheight}{10in} \setlength{\textwidth}{6.875in}
\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{-0.5in}
\setlength{\headheight}{0in} \setlength{\headsep}{0in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.2in}  % Centers text.
\setlength{\evensidemargin}{-.2in}

% Add the period after section numbers.  Adjust spacing.

\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand{\bq}{\begin{eqnarray*}}
\newcommand{\eq}{\end{eqnarray*}}
\begin{document}


% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 312: Lecture 05\\
Maximum Likelihood Estimation}

% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
\date{September 14, 2004}
\maketitle \thispagestyle{empty}
\begin{enumerate}

\item (Invariance Principle) If $\hat \theta$ is the MLE's of parameter $\theta$ then the MLE of $h(\theta)$ is $h(\hat \theta)$ for some function
$h$.\\
{\em Proof} (partial). Consider likelihood function $L(\theta)$.
$\hat \theta$ satisfies $\frac{d L(\theta)}{d \theta}=0$. Let $\phi
= h(\theta)$. Then the likelihood function for $\phi=h(\theta)$ is
given by $L(h^{-1}(\phi))$. Differentiating the likelihood with
respect to $\phi$, we have
$$\frac{L(h^{-1}(\phi))}{d\phi} = \frac{d L(\theta)}{d \theta}
\frac{d \theta}{d \phi} =\frac{d L(\theta)}{d \theta}\frac{1}{
h'(\theta)}=0.$$


\item {\em Loglikelihood.} Maximizing $L(\theta)$ is equivalent to maximizing $\ln
L(\theta)$ since $\ln$ is an increasing function.\\
Example. This technique is best illustrated by finding the MLE of
parameters in $N(\mu,\sigma^2)$. $$\hat \mu = \bar X,$$ $$\hat
\sigma^2 = \frac{1}{n}\sum_{i=1}^n (X_i - \bar X)^2$$ are the MLE of
$\mu$ and $\sigma^2$ respectively. Note that $\hat \sigma^2$ is not
un unbiased estimator of $\sigma^2$.

\item {\em Asymptotic unbiasness.} When the sample size is large, the maximum likelihood
estimator of $\theta$ is approximately unbiased. The MLE of $\theta$
is approximately the MVUE of $\theta$. This is why it is the most
widely used estimation technique in statistics. For the previous
example, $$\mathbb{E} \hat \sigma^2 = \mathbb{E} \frac{n-1}{n} S^2 =
\frac{n-1}{n} \sigma^2 \to \sigma^2 \mbox{ as } n \to \infty$$




\item If explicit density function is not available, you can not apply MLE. In this case apply the method of moment matching.



\item {\em Additional problem (previous midterm).} Let $X_1 ,X_2$ be a random sample
from $N(0, 1/\theta)$. Note that the sample size is 2 and the
density function for $X_i$ is $$f(x_i) =
\frac{\sqrt{\theta}}{\sqrt{2\pi}}\exp(-\theta x_i^2/2).$$ Find the
likelihood function and use it to obtain the maximum likelihood
estimator of $\theta$.\\
{\em Solution.} The likelihood function is $$L(\theta)
=\theta/(2\pi)\exp \Big(-\theta(x_1^2 +x_2^2)/2\Big).$$ Now get
log-likelihood function $$\ln L(\theta) = const + \ln \theta -
\theta (x_1^2+x_2^2)/2.$$ Differentiate with respect to $\theta$ we
get
$$\frac{d \ln L(\theta)}{d\theta} = \frac{1}{\theta} - \frac{1}{2}(x_1^2+x_2^2)=0.$$
Solving the equation, we get $$\hat \theta = 2/(x_1^2 + x_2^2).$$

\end{enumerate}


{\em Review problems.} Example 6.17., Example 6.18., Exercise 6.23.,
Exercise 6.29. Read Chapter 7.

{\bf Homework II.} Exercise 6.30., 6.38., 7.4. 7.10. 7.12. 7.14. Due
Sept 30. 9:30am.




\end{document}