% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.
% Make two column format for LaTex 2e.\
\documentclass[12pt,twocolumn]{article} %,twocolumn
\usepackage{times,amsmath,amsfonts}

% Use following instead for LaTex 2.09 (may need some other mods as well).
%\documentstyle[times,twocolumn]{article}
\usepackage[dvips]{graphicx,graphics}
% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{11in} \setlength{\textwidth}{7in}
\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.5in}  % Centers text.
\setlength{\evensidemargin}{-.5in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-8pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-3pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand {\diff}[1] {\frac{\partial}{\partial #1}}
\newcommand{\jacob}[3]{\frac{\partial^2 #3}{\partial #1 \partial #2}}
\newcommand{\der}[2]{\frac{\partial #2}{\partial #1}}
\begin{document}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 471: Lecture 19\\
Markov Chains II.}
% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
% For two authors (default example)
\maketitle \thispagestyle{empty}
\begin{enumerate} 

\item A rat became insane and moves back and forth between position 1 and 2. Let $X_i$ be the
position of the rat at the $i$-th move. Suppose that the transition probability is given by 
$${\bf P}=\left(
\begin{array}{cc}
  \frac{1}{2} & \frac{1}{2} \\
  1 & 0 \\
\end{array}
\right).$$ 
A state $i$ is called {\em recurrent} if the Markov chain returns to $i$ with probability 1 in a finite number of steps otherwise the state is {\em transient}. The recurrent of a state is equivalent to a guarantee of a sure return. Is the state $1$ and $2$ recurrent?
$$\sum_{i=2}^{\infty} P(X_{i}=j|X_0=j,X_1 \neq j,\cdots X_{i-1}, \neq j)$$
$$=\frac{1}{2}+\frac{1}{2^2} + \cdots  =1$$
 for $j=1,2$. So the state $1$ and $2$ are recurrent.

\item Let us find $P(X_n=1|X_0=2)$. Note that this is $n$ step transition probability denoted by $P_{12}^n$. It is computed by the Chapman-Kolmogorov equation. For this we need to compute ${\bf P}^n$. In general you can compute this by the Cayley-Hamilton theorem and solving $n$-th order finite difference equation. But why don't we skip it but basically you can compute it numerically in $\tt{MATLAB}$.
\begin{verbatim}
P=[1/2 1/2
     1 0]
>>P^2
ans =
  0.75000  0.25000
  0.50000  0.50000 
>> P^5
ans =
  0.65625  0.34375
  0.68750  0.31250
>> P^100
ans =
  0.66667  0.33333
  0.66667  0.33333
>> P^1000000000
ans =
  0.66667  0.33333
  0.66667  0.33333
\end{verbatim}
So we can see that the transition probabilities converge: $\lim_{n \to \infty} P_{ij}^{n} =\pi_j$. $\pi_1=\frac{2}{3}$ and $\pi_2=\frac{1}{3}$.
\item A Markov chain is stationary or homogeneous if $(X_0,\cdots,X_i) \sim (X_n,\cdots,X_{n+i})$. Suppose we have state space $S$ as the sample space and let $\pi =\{\pi_j: j \in S\}$ be a probability distribution defined on $S$, i.e. $\sum _{j \in S} \pi_j =1$. Then $\pi$ is a stationary distribution for the Markov chain with transition probability ${\bf P}$ if $\pi' = \pi{\bf P}$. Note that
$$\pi' = \pi'{\bf P} = \pi'{\bf P}^2 = \cdots = \pi'{\bf P}^n.$$
This probability $\pi$ is also called {\em invariant probability measure}. Read S.I. Resnick's Adventures in Stochastic Processes for theoretical details on Markov Chains and a stationary distribution.
A couple of interesting properties on $\pi_j$. $\frac{1}{\pi_j}$ is the expected number of states a Markov chain should take to return to the original state $j$. In our crazy rat example, the rat will return to position 2 in average 3 steps if it was at the position 2 initially.

\item An increased stability for a Markov chain can be achieved if we let $X_0 \sim \pi$, i.e. $P(X_0=j) = \pi_j$. From the total low of probability
$P(A)=\sum_{i=1}^{\infty} P(B_i)P(A|B_i)$ if $\{ B_i \}$ partition $S$.
So $P(X_1=j) = \sum_{i \in S} \pi_i P_{ij}$. Hence $X_1 \sim \pi'{\bf P} =\pi'.$
Similarly $X_i \sim \pi$. MCMC is based on this fact and requirement.

\end{enumerate}
\end{document}