% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.
% Make two column format for LaTex 2e.\
\documentclass[12pt,twocolumn]{article} %,twocolumn
\usepackage{times,amsmath,amsfonts}

% Use following instead for LaTex 2.09 (may need some other mods as well).
%\documentstyle[times,twocolumn]{article}
\usepackage[dvips]{graphicx,graphics}
% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{10in} \setlength{\textwidth}{7in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.5in}  % Centers text.
\setlength{\evensidemargin}{-.5in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-8pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-3pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand {\diff}[1] {\frac{\partial}{\partial #1}}
\newcommand{\jacob}[3]{\frac{\partial^2 #3}{\partial #1 \partial #2}}
\newcommand{\der}[2]{\frac{\partial #2}{\partial #1}}
\begin{document}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 471: Lecture 18\\
Markov Chains}
% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
% For two authors (default example)
\maketitle \thispagestyle{empty}
\begin{enumerate} 

\item A stochastic process is a collection of random variables $X_i$ indexed over some set $A$, i.e. $\{ X_i: i \in A \}$.  If $A$ is a discrete set, we have a discrete stochastic process. If $A$ is continuous, we have a continuous stochastic process. Let $X_0,X_1,\cdots$ be a stochastic process. A Markov chain is a sequence of random variables such that the next state $X_{i+1}$ depends only on the current state $X_i$ (memoryless property), i.e. $$P(X_{i+1}=y|X_i=x_i,\cdots,X_0=x_0)$$ 
$$=P(X_{i+1}=y|X_i=x_i).$$
The probability $P_{xy}=P(X_{i+1}=y|X_i=x)$ is called the {\em transition kernel}. Note that $\sum_y P_{xy}=1$. The initial distribution for $X_0$ determine the distribution for any $n$-th state.
\end{verbatim}
\begin{figure}
\centering
\includegraphics[scale=0.4]{lecture18-1.eps}
\end{figure}
\item Random walk. You play a coin tossing game with me. You win $\$ 1$ if a head appears and lose $\$ 1$ if a tail appears. Let $X_i$ be your total gain after $i$ tosses. Assume the coin to be unbiased. Then the transition kernel is given by
$$P(X_{i+1}= x+1|X_i = x) =\frac{1}{2},$$
$$P(X_{i+1}= x-1|X_i = x) =\frac{1}{2}.$$
We can model the coin tossing a simple random walk
$X_{i+1} = X_i + \epsilon_i,$
with $P(\epsilon_i = 1)=\frac{1}{2}$ and $P(\epsilon_i = -1) = \frac{1}{2}$.
The expected pay off does not change over time, i.e. $\mathbb{E} X_{i+1} = \mathbb{E} X_i.$
\begin{verbatim}
x(1)=0; n=1000
e = (-1).^(rand(n,1) < 0.5)
for i=1:(n+1)
  x(i+1)=x(i) + e(i);
end;
\end{verbatim}
In the case of random walk, $X_{i+1} = \sum_{j=0}^n \epsilon_j + X_0$. Then $P(X_{n+1}=x|X_0=0)=P(\sum_{j=0}^n \epsilon_j = x)$. Compute this probability (HW 4).

\item Chapman-Kolmogorov equation. Let $P_{xy}^n$ be the transition kernel from $x$ to $y$ in $n$-steps, i.e. $P_{xy}^n=P(X_{i+n}=y|X_i=x)$. Then we have
$$P_{xy}^{m+n} = \sum_{z} P_{xz}^m P_{zy}^n.$$
The the probability of reaching $y$ from $x$ in $n$-steps is the sum of all probabilities going from $x$ to $y$ through an intermediate point $z$. Let ${\bf P}^n = (P_{ij}^n)$ be a  matrix. Then in terms of matrix, Chapman-Kolomogorov equation can be trivially written as ${\bf P}^{m+n} = {\bf P}^m {\bf P}^n$.

\end{enumerate}

\end{document}