% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.

% Make two column format for LaTex 2e.\
%\documentclass[10pt,twocolumn]{article} %,twocolumn
\documentclass[12pt,twocolumn]{article} %,twocolumn


\usepackage{times,amsmath,amsfonts}
\usepackage[dvips]{graphicx,graphics}
% Use following instead for LaTex 2.09 (may need some other mods as well).
%\documentstyle[times,twocolumn]{article}

% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{10in} \setlength{\textwidth}{6in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}

% Add the period after section numbers.  Adjust spacing.

\setlength{\textheight}{10.5in} \setlength{\textwidth}{7.6in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-0.6in}  % Centers text.
\setlength{\evensidemargin}{-0.5in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-5pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-2pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray*}}
\newcommand{\eqn}{\end{eqnarray*}}
\newcommand{\bq}{\begin{eqnarray}}
\newcommand{\eq}{\end{eqnarray}}


\begin{document}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 471: Lecture 32\\
Gaussian Mixture and EM algorithm}
% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
% For two authors (default example)
\maketitle \thispagestyle{empty}


%\begin{figure}
%\centering
%\renewcommand{\baselinestretch}{1}
%\includegraphics[scale=0.8]{homework05-1.eps}
%\end{figure}

\begin{enumerate}

\item The image intensity can be modelled as a Gaussian mixture of
the form
$$f(y) = p f_1(y) + (1-p)f_2(y)$$ where $f_1 \sim N(\mu_1,\sigma_1^2)$ and $f_2 \sim
N(\mu_2,\sigma_2^2)$. The likelihood function is
$$f(p|y)=\prod_{i=1}^n [pf_1(y_i)+qf_2(y_i)]$$
where $q=1-p$. The loglikelihood is
$$L(p|y) = \sum_{i=1}^n \log[pf_1(y_i)+qf_2(y_i)].$$
Solving
$$\frac{\partial L(p|y)}{\partial p}=0$$
to maximize the loglikelihood is complicated. So we argument data
with missing data and apply the EM algorithm.

\item Let $X$ be a Bernoulli random variable with $P(X=1)=p$. Let
$Y \sim f_1$ if $X=1$ and $Y \sim f_2$ if $X=0$. Then the joint
density $f(x,y)$ of $(X,Y)$ is $f(1,y) = pf_1(y)$ and
$f(0,y)=qf_2(y)$. This can be written simply as
$$f(x,y) = [pf_1(y)]^x[qf_2(y)]^{1-x}.$$
The marginal density of $Y$ is obviously
$$f(y) = pf_1(y) + qf_2(y).$$
The conditional density of $X$ given $Y$ is then
$$f(x|y) = \frac{[pf_1(y)]^x[qf_2(y)]^{1-x}}{ pf_1(y) + qf_2(y)}.
$$
Trivially the conditional expectation \bq\mathbb{E}(X|y,p) =
\frac{pf_1(y)}{pf_1(y) + qf_2(y)}.\label{eq:32-1}\eq
 For each observation $Y_i$, we argument it with missing data
$X_i$ which is Bernoulli with the above property. The likelihood
for the complete data $(x,y)$ is
$$f(p|x,y) = \prod_{i=1}^n [pf_1(y_i)]^{x_i}[qf_2(y_i)]^{1-x_i}$$
So the loglikelihood for the complete data is
\bqn &&L(p|x,y)\\
 &=& \sum_{i=1}^n x_i\log [p f_1(y_i)]
 + (1-x_i)\log[(qf_2(y_i)] \eqn

\item Now we take expectation and get $Q$ function
\bqn & &Q(p|p_0,y)\\
 &=&\mathbb{E}[\log L(p|X,Y)|y,p_0]\\
&&\sum_{i=1}^n \mathbb{E}(X_i|y,p_0)
\log[\frac{pf_1(y_i)}{qf_2(y_i)}\Big]
 +\log [qf_2(y_i)] \eqn

From equation (\ref{eq:32-1}),
$$\mathbb{E}(X_i|y,p_0) = \frac{p_0f_1(y_i)}{ p_0f_1(y_i) + q_0
f_2(y_i)}.$$ Neglecting parts that do not contain $p$,
 \bqn
Q(p|p_0,y) &=& \sum_{i=1}^n \frac{p_0f_1(y_i)}{ p_0f_1(y_i) + q_0
f_2(y_i)}\log \frac{p}{1-p}\\
&&+ n\log (1-p).\eqn Maximizing $Q$ by $\partial Q/\partial p=0$,
we get $$p= \frac{1}{n} \sum_{i=1}^n \frac{p_0f_1(y_i)}{
p_0f_1(y_i) + q_0 f_2(y_i)}.$$ Based on this, we set up iteration
$$\hat p_{j+1} = \frac{1}{n} \sum_{i=1}^n \frac{\hat p_jf_1(y_i)}{
\hat p_jf_1(y_i) + (1-\hat p_j) f_2(y_i)}$$ with any initial
estimate $\hat p_0$. Setting $\hat p_0=0.5$ will speed up the
convergence. Cool?

 %Assume $\mu_1=170,\sigma^2=20^2, \mu_2=210, \sigma_2^2=10^2$.
%Estimate parameter $0<p<1$ by maximizing the likelihood function
%using the EM algorithm. Use both deterministic and Monte-Carlo
%versions. What this parameter $p$ measure?

\end{enumerate}
\end{document}