% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.

% Make two column format for LaTex 2e.


\documentclass[10pt,twocolumn]{article} %
\usepackage[dvips]{graphicx,graphics,color}
\usepackage{times,amsmath,amsfonts}
%\usepackage{pancyhdr}
\setlength{\textheight}{10.5in} \setlength{\textwidth}{7in}
\setlength{\columnsep}{0.3in} \setlength{\topmargin}{-0.5in}
\setlength{\headheight}{-0.2in} \setlength{\headsep}{0in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.2in}  % Centers text.
\setlength{\evensidemargin}{-.2in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-8pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-3pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}


\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand{\bq}{\begin{eqnarray*}}
\newcommand{\eq}{\end{eqnarray*}}


\begin{document}

% Don't want date printed
% Make title bold and 14 pt font (Latex default is non-bold, 16pt)

\title{Stat 311-Lecture 14 Moment generating functions}
% For single author (just remove % characters)
%\author{\Large Probability}
\vspace{-2cm}
\date{ }
% For two authors (default example)
\maketitle \thispagestyle{empty}

Dec 5, 2005


\begin{enumerate}

\item  For random variable
$X$, the moment generating function of $X$ is a function of $t$
given by
$$M_X(t) = \mathbb{E} e^{tX}$$
for any  $|t| \leq c$. This function can be used in determining any
$n$-th moment automatically. The Taylor expansion of $e^{tX}$ is
$$ e^{tX} = 1 + t X + \frac{t^2}{2!}X^2 +  \frac{t^3}{3!}X^3 \cdots.
$$
Taking the expectation on the both side, we have
$$M_X(t) = 1 + t\mathbb{E} X + \frac{t^2}{2!}\mathbb{E} X^2 +  \frac{t^3}{3!}\mathbb{E}X^3
\cdots.$$ Hence the series expansion of the moment generating
function consists of the moments of $X$. By differentiating the
moment generating function and letting $t=0$, we have
$$M_X'(0) = \mathbb{E}X,$$
$$M_X''(0) = \mathbb{E}X^2.$$
So the variance can be obtained as $$\mathbb{V} X = M_X''(0)
-(M_X'(0))^2.$$

\item For $X_i \sim Bernoulli(p)$, the moment generating function is
given by $M_{X_i}(t) = 1-p + pe^t$.

\item For iid $X_1,X_2,\cdots,X_n$, the moment generating functions
of $S_n = X_1 + X_2 + \cdots + X_n$ is given by
$$M_{S_n}(t) = \mathbb{E} e^{t(X_1 + X_2 + \cdots + X_n)} = M_{X_1}^n(t).$$
Using this property, the moment generating for $X \sim
Binomial(n,p)$ is given by $M_{X}(t) = (1-p + pe^t)^n$. Note that
$M_X'(t) = n(1-p+pe^t)^{n-1}pe^t$. So $\mathbb{E} X = M_X'(0)=np$.


\item Random variables $X$ and $Y$ have the same distribution if
$M_X(t) = M_Y(t)$ for all possible values of $t$. This property can
be used to show that the sum of independent binomial distributions
is again binomial. Consider independent $X \sim Binomial(n_1,p)$ and
$Y \sim Binomial(n_2,p)$. The moment generating functions are
$M_X(t) = ((1-p + pe^t)^{n_1}$ and $M_Y(t) = ((1-p + pe^t)^{n_2}$.
The moment generating function for $X+Y$ is
$$M_{X+Y}(t)=M_{X}(t)M_{Y}(t) = (1-p + pe^t)^{n_1}(pe^t+q)^{n_2}.$$
Since this is the moment generating function of
Binomial$(n_1+n_2,p)$, $X+Y$ must be distributed as
Binomial$(n_1+n_2,p)$.

\item Suppose that $X$ and $Y$ are independent and identically distributed random variables whose density is of the form $f_X(x)=\frac{x^{\alpha-1}e^{-x}}{\Gamma(\alpha)}, \alpha >0, x>0.$ Evaluate the $n-$th moment of $X$ and find the density of $X+Y$. \\
{\it solution.} $X$ and $Y$ are gamma distributions. Directly we can
compute \bq \mathbb{E}X^n &=& \int_0^\infty
x^n\frac{x^{\alpha-1}e^{-x}}{\Gamma(\alpha)} dx \\
&=& \frac{\Gamma (n+\alpha)}{\Gamma (\alpha)}\int_0^{\infty}
\frac{x^{n+\alpha-1}e^{-x}}{\Gamma(n+\alpha)} dx =
\frac{\Gamma(n+\alpha)}{\Gamma(\alpha)}.\eq


Using the moment generating function method, we have $$M_X(t)=
\int_0^\infty e^{tx}\frac{x^{\alpha-1}e^{-x}}{\Gamma(\alpha)} dx =
\int_0^\infty \frac{x^{\alpha-1}e^{-(1-t)x}}{\Gamma(\alpha)} dx.$$
Let $y= (1-t)x$. Then \bq M_X(t) &=&  \int_0^\infty
\Big(\frac{y}{1-t}\Big)^{\alpha-1}\frac{e^{-y}}{\Gamma(\alpha)}
\frac{dy}{1-t}\\
&=& \frac{1}{(1-t)^{\alpha}}\int_0^{\infty}
\frac{y^{\alpha-1}e^{-y}}{\Gamma(\alpha)} dy
=\frac{1}{(1-t)^{\alpha}}.\eq Since $X$ and $Y$ are i.i.d.,
$M_{X+Y}(t) = M_X(t)M_Y(t) = \frac{1}{(1-t)^{2\alpha}}$, indicating
that the density of $X+Y$ should have the same form as the density
of $X$ with different parameter $2\alpha$. Hence,
$$f_{X+Y}(v) = \frac{v^{2\alpha-1}e^{-v} }{\Gamma(2\alpha)}, 0 < v.$$ The sum of independent gamma distributions are gamma.

\item For independent $X,Y \sim N(\mu,\sigma^2)$, proving the normality of $\alpha X + \beta
Y$ is similar to the previous example. 

\end{enumerate}



\end{document}