% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.
% Make two column format for LaTex 2e.\
\documentclass[12pt,twocolumn]{article} %,twocolumn
\usepackage{times,amsmath,amsfonts}

% Use following instead for LaTex 2.09 (may need some other mods as well).
%\documentstyle[times,twocolumn]{article}
\usepackage[dvips]{graphicx,graphics}
% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{10.5in} \setlength{\textwidth}{7.6in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-0.6in}  % Centers text.
\setlength{\evensidemargin}{-0.5in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-5pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-2pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}

\begin{document}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 471: Lecture 26\\
Expectation-Maximization Algorithm I.}
% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
% For two authors (default example)
\maketitle \thispagestyle{empty}
\begin{enumerate}
\item {\em Expectation-maximization} (EM) method is an iterative method for maximizing  difficult likelihood problems. It was first introduced by Dempster {\em et al.} (J. Roy. Statist. Soc. 1997). Suppose we have a random sample $X=(X_1,\cdots, X_n)$ iid from $f(x|\theta)$. We wish to find the maximum likelihood estimator 
$$\hat \theta = \arg \max_{\theta} \prod_{i=1}^n f(x_i|\theta) = \arg \max_{\theta} \sum_{i=1}^n \ln f(x_i|\theta).$$ 
This optimization problem is difficult and it gives a motivation for EM algorithm.  
We augment the data with with {\em latent} (unobserved or missing) data $X^m$ such that the
complete data $X^c = (X, X^m)$. The density of $X^c$ is 
$$X^c=(X,X^m) \sim f(x^c) = f(x,x^m).$$ The conditional density for the missing data $X^m$ is
$$f(x^m|x,\theta) = \frac{f(x,x^m|\theta)}{f(x|\theta)}.$$
Rearranging terms,
$$f(x|\theta) = \frac{f(x,x^m|\theta)}{f(x^m|x,\theta)}.$$
Taking logarithm, we get log-likelihood
$$\ln f(X|\theta) = \ln f(X^c|\theta) - \ln f(X^m|X,\theta).$$
Take expectation with respect to $f(x^m|x,\theta_0)$ so that $X$ are considered as constants. 

\bqn \ln f(X|\theta) &=& \mathbb{E} [\ln f(X^c|\theta) |X,\theta_0]\\
&& - \mathbb{E} [\ln f(X^m|X,\theta)|X,\theta_0].
\eqn


\item. Let us denote the log-likelihood by
$$Q(\theta|\theta_0,x) = \mathbb{E} [\ln f(X^c|\theta)|X,\theta_0].$$
{\bf EM algorithm}\\
1. the E-step: compute $Q(\theta|\hat \theta_{j-1},x)$\\
2. the M-step: maximize $Q(\theta|\hat \theta_{j-1},x)$ and take
$$\hat \theta_j = \arg \max_{\theta} Q(\theta|\hat \theta_{j-1},x).$$
{\em Proof.} If the above procedure is iterated, we get the sequence of estimators
$\hat \theta_0,\hat \theta_1, \cdots $ and it can be shown that it converges to
the maximum likelihood estimator $\hat \theta$.\\ 


Note that $Q(\hat \theta_{j+1}|\hat \theta_j,x) \geq Q(\hat \theta_j|\hat \theta_j,x)$. Now let 
$$R(\theta|\theta_0,x) = 
\mathbb{E} [\ln f(X^m|X,\theta)|X,\theta_0].$$
It can be shown that $\theta_0 = \arg \max_{\theta} R(\theta|\theta_0,x)$ For the proof, use Jensen's inequality (see Carsella $\&$ Berger's  Statistical Inference). Then
$$ R(\hat \theta_{j+1}|\hat \theta_j,x) \leq R(\hat \theta_j|\hat \theta_j,x).$$
Consequently
$$\ln f(X|\hat \theta_j) \leq \ln f(X|\hat \theta_{j+1}).$$
This inequality guarantee the the sequence of estimators $\hat \theta_j$ monotonically increase the likelihood. To guarantee that the limit converges to the maximum likelihood estimator, we need the condition of continuity of $Q(\theta|\theta_0,x)$ in $\theta$ and $\theta_0$.


\item The difficulty of EM algorithm is at the E-step where we need to compute the conditional expectation $Q(\theta|\hat \theta_{j-1},x)$. The {\em Monte-Carlo EM} algorithm overcome this by simulating missing data $X^m \sim f(x^m|x,\theta)$. 
so that $$\widehat Q(\theta|\theta_0,x) = \frac{1}{l}\sum_{j=1}^l \ln f(X,X^m|\theta).$$
\end{enumerate}
\end{document}