% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.

% Make two column format for LaTex 2e.
\documentclass[11pt,twocolumn]{article} %
\usepackage[dvips]{graphicx,graphics,color}
\usepackage{times,amsmath,amsfonts}

\setlength{\textheight}{10in} \setlength{\textwidth}{6.875in}
\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{-0.5in}
\setlength{\headheight}{0in} \setlength{\headsep}{0in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.2in}  % Centers text.
\setlength{\evensidemargin}{-.2in}

% Add the period after section numbers.  Adjust spacing.

\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand{\bq}{\begin{eqnarray*}}
\newcommand{\eq}{\end{eqnarray*}}
\begin{document}


% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 324: Lecture 15\\
Maximum likelihood estimation}

% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
\date{March 17, 2005}
\maketitle \thispagestyle{empty}
\begin{enumerate}

\section*{Concepts}



\item Given random sample $X_1,\cdots,X_n$,
the {\em likelihood function} is given as the product of probability
or density functions, i.e.
$$L(\theta)= f(x_1;\theta)f(x_2;\theta)\cdots f(x_n;\theta).$$
The {\em maximum likelihood estimatate} of $\theta$ is an estimate
that
 maximizes $L(\theta)$. If we denote $\hat \theta=\theta (x_1,\cdots,x_n)$ to
be the maximum likelihood estimate, the {\em maximum likelihood
estimator} (MLE) of $\theta$ is $\hat \theta = \hat
\theta(X_1,\cdots,X_n)$.

\item {\em Exponential distribution.} $X$ is an
exponential distribution with parameter $\lambda$ if the density
function is $f(x) = \lambda e^{-\lambda x}$ for $ x \geq 0$. It can
be shown that $\mathbb{X}= \frac{1}{\lambda}, \mathbb{V}X =
\frac{1}{\lambda^2}$. Then MLE of $\lambda$ is $\hat \lambda = \bar
X$.



\item {\em Loglikelihood.} Maximizing $L(\theta)$ is equivalent to maximizing $\ln
L(\theta)$ since $\ln$ is an increasing function.\\
Example. This technique is best illustrated by finding the MLE of
parameters in $N(\mu,\sigma^2)$. $$\hat \mu = \bar X,$$ $$\hat
\sigma^2 = \frac{1}{n}\sum_{i=1}^n (X_i - \bar X)^2$$ are the MLE of
$\mu$ and $\sigma^2$ respectively. Note that $\hat \sigma^2$ is not
un unbiased estimator of $\sigma^2$.


\item (Invariance Principle) If $\hat \theta$ is the MLE's of parameter $\theta$ then the MLE of $h(\theta)$ is $h(\hat \theta)$ for some function
$h$.\\
{\em Proof.} Consider likelihood function $L(\theta)$. $\hat \theta$
satisfies $\frac{d L(\theta)}{d \theta}=0$. Let $\phi = h(\theta)$.
Then the likelihood function for $\phi=h(\theta)$ is given by
$L(h^{-1}(\phi))$. Differentiating the likelihood with respect to
$\phi$, we have
$$\frac{L(h^{-1}(\phi))}{d\phi} = \frac{d L(\theta)}{d \theta}
\frac{d \theta}{d \phi} =\frac{d L(\theta)}{d \theta}\frac{1}{
h'(\theta)}=0.$$

\item {\em Asymptotic unbiasness.} When the sample size is large, the maximum likelihood
estimator of $\theta$ is approximately unbiased. The MLE of $\theta$
is approximately the MVUE of $\theta$. This is why it is the most
widely used estimation technique in statistics. For the previous
example, $$\mathbb{E} \hat \sigma^2 = \mathbb{E} \frac{n-1}{n} S^2 =
\frac{n-1}{n} \sigma^2 \to \sigma^2 \mbox{ as } n \to \infty$$




%\item If explicit density function is not available, you can not apply MLE. In this case apply the method of moment matching.



\section*{Problems} Let $X_1 ,X_2$ be a random sample from $N(0, 1/\theta)$.
Note that the sample size is 2 and the density function for $X_i$ is
$$f(x_i) = \frac{\sqrt{\theta}}{\sqrt{2\pi}}\exp(-\theta x_i^2/2).$$
Find the likelihood function and use it to obtain the maximum
likelihood
estimator of $\theta$.\\
{\em Solution.} The likelihood function is $$L(\theta)
=\theta/(2\pi)\exp \Big(-\theta(x_1^2 +x_2^2)/2\Big).$$ Now get
log-likelihood function $$\ln L(\theta) = const + \ln \theta -
\theta (x_1^2+x_2^2)/2.$$ Differentiate with respect to $\theta$ we
get
$$\frac{d \ln L(\theta)}{d\theta} = \frac{1}{\theta} - \frac{1}{2}(x_1^2+x_2^2)=0.$$
Solving the equation, we get $$\hat \theta = 2/(x_1^2 + x_2^2).$$

\end{enumerate}
{\bf Assigned problems.} 6.20., 6.30. HW 5 due April 14th. Solve the
assigned problems in Lecture 15-18. 



\end{document}