% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.
% Make two column format for LaTex 2e.\
\documentclass[12pt,twocolumn]{article} %,twocolumn
\usepackage{times,amsmath,amsfonts}

% Use following instead for LaTex 2.09 (may need some other mods as well).
%\documentstyle[times,twocolumn]{article}
\usepackage[dvips]{graphicx,graphics}
% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{11in} \setlength{\textwidth}{7.5in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-0.6in}  % Centers text.
\setlength{\evensidemargin}{-0.5in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-8pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-3pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand {\diff}[1] {\frac{\partial}{\partial #1}}
\newcommand{\jacob}[3]{\frac{\partial^2 #3}{\partial #1 \partial #2}}
\newcommand{\der}[2]{\frac{\partial #2}{\partial #1}}
\begin{document}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 471: Lecture 21\\
Markov Chain Monte Carlo I.}
% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
% For two authors (default example)
\maketitle \thispagestyle{empty}
\begin{enumerate} 

\item  Suppose we have state space $S$ as the sample space and let $\pi =\{\pi_j: j \in S\}$ be a probability distribution defined on $S$, i.e. $\sum _{j \in S} \pi_j =1$. Then $\pi$ is a stationary distribution for the Markov chain with transition probability ${\bf P}$ if $\pi' = \pi{\bf P}$. Note that
$$\pi' = \pi'{\bf P} = \pi'{\bf P}^2 = \cdots = \pi'{\bf P}^n.$$
This probability $\pi$ is also called {\em invariant probability measure}. Read S.I. Resnick's Adventures in Stochastic Processes for theoretical details on Markov Chains and a stationary distribution.
A couple of interesting properties on $\pi_j$. $\frac{1}{\pi_j}$ is
the expected number of states a Markov chain should take to return to
the original state $j$ (the mean recurrence time). In our crazy rat example, the rat will return to position 2 in average 3 steps if it was at the position 2 initially.

\item If we let $X_0 \sim \pi$, i.e. $P(X_0=j) = \pi_j$. From the law of total probability
$P(A)=\sum_{i=1}^{\infty} P(B_i)P(A|B_i)$ if $\{ B_i \}$ partition $S$.
So $P(X_1=j) = \sum_{i \in S} \pi_i P_{ij}$. Hence $X_1 \sim \pi'{\bf P} =\pi'.$
Similarly $X_i \sim \pi$ for all $i$.

\item Actually you don't need to have $X_0 \sim \pi$ to have a stable chain. If $X_0 \sim \mu$ for any probability distribution, $\mu'{\bf P}^n \to \pi'$ as $n \to \infty$. For proof, see G. Winkler's Image Analysis, Random Fields and Markov Chain Monte Carlo Methods Theorem 4.3.1.
\begin{verbatim}
>> mu' %generate from rand
  0.534161  0.465839
  0.986845  0.013155
  0.969502  0.030498
>> mu'*P
  0.73292  0.26708
  0.50658  0.49342
  0.51525  0.48475
>> mu'*P^10
  0.66654  0.33346
  0.66698  0.33302
  0.66696  0.33304
>>mu'*P^100
  0.66667  0.33333
  0.66667  0.33333
  0.66667  0.33333
\end{verbatim}
\item Ergodic Theorem. Suppose a Markov chain $X_i$ with kernel ${\bf P}$ on a finite sample space $S$ has invariant distribution $\pi$ then for any distribution $\mu$ and function $g$, 
$$\lim_{n \to \infty} \frac{1}{n}\sum_{i=0}^n g(X_i) = \mathbb{E}_{\pi} g = \sum_{i \in S} g(i)\pi_i.$$
After a large number of run , say $m$, the above convergence can be written
$$\mathbb{E}_{\pi} g \doteq \frac{1}{n-m}\sum_{i=m+1}^n g(X_i).$$
The number of samples $m$ that are discarded for the above estimation is called the {\em burn in}. This is the basis of MCMC. Given probability distribution $\pi$, we estimate $\mathbb{E}_{\pi} g$ by constructing a Markov chain $X_i$ and computing the ergodic limit. 

\item The difference between MCMC and a simple Monte-Carlo method is if $X_i$ are iid. Generalized by Hastings (1970), the Metropolis-Hastings algorithm is the only known method of MCMC. All other MCMC-looking procedures are actually pseudo-MCMC. The number $n$ can be determined by running several Markov chains in parallel, each with a different initial value. If the estimated $\mathbb{E}_{\pi} g$ has large variation, increase the number of chains. 
\end{enumerate}
\end{document}