% $Header: /cvsroot/latex-beamer/latex-beamer/examples/beamerexample5.tex,v 1.22 2004/10/08 14:02:33 tantau Exp $

\documentclass[11pt]{beamer}
\usepackage{amscd}

%\usetheme{Madrid} % Darmstadt, Boadilla


\mode<presentation> {
  \usetheme{Madrid}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsbsy,alltt,amsmath,times}
\usepackage{epsfig, color, graphicx, rotate, colordvi}
%\usepackage{fancybox,fancyhdr,times,subfigure}
\usepackage{hyperref, mathptmx}


\setbeamercovered{dynamic}

\newcommand{\Lang}[1]{\operatorname{\text{\textsc{#1}}}}

\newcommand{\Class}[1]{\operatorname{\mathchoice
  {\text{\sf \small #1}}
  {\text{\sf \small #1}}
  {\text{\sf #1}}
  {\text{\sf #1}}}}

\newcommand{\NumSAT}      {\text{\small\#SAT}}
\newcommand{\NumA}        {\#_{\!A}}

\newcommand{\barA}        {\,\bar{\!A}}

\newcommand{\Nat}{\mathbb{N}}
\newcommand{\Set}[1]{\{#1\}}

\newtheorem{thm}{Result}%[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{defn}[thm]{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{rem}{Remark}

\newcommand{\cred}{\textcolor {red}}
\newcommand{\cblue}{\textcolor {blue}}
\newcommand{\cgreen}{\textcolor {green}}

\def\ul{\underline}

\def\sign{\;{\rm sign}}
\def\rank{\;{\rm rank}}
\def\disp{\displaystyle}
\def\sn{\sqrt{n}}
\def\Pr{\mathrm{P}}

\def\Cov{\;{\rm Cov}}
\def\L{\underline L}
\def\LL{\overline L}

\def\balpha{{\boldsymbol{\alpha}}}
\def\cal{\mathcal}
\def\conas{\stackrel{a.s.} {\longrightarrow}}         % conv. a.s.
\def\conp{\stackrel{\cal P} {\longrightarrow}}        % conv. in probability
\def\conD{\stackrel{\mathcal{L}} {\longrightarrow}}        % conv. in distribution
\def\conw{\stackrel{\mathcal{L}} {\rightarrow}}        % conv. in distribution
\def\hat{\widehat}
%\def\endproof{\begin{flushright}$\lozenge$\end{flushright}}
\def\Proof{\noindent{\it Proof }}
\def\endproof{\hfill{$\Box$}}
\newcommand{\wcon}{\stackrel{\mathcal{L}}{\longrightarrow}}
\def\Var{\mbox{var}}
\def\br{\begin{array}}
\def\er{\end{array}}

%%%%%%%%%%%%%%% MY DEFINITIONS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Put your own definitions
%
\def\VaR{{\rm VaR}}
\def\E{{\rm E}}
\def\beqrs{\begin{eqnarray*}}           %\beqrs
\def\eeqrs{\end{eqnarray*}}             %\eeqrs

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\pgfdeclaremask{tu}{beamer-tu-logo-mask}
\pgfdeclaremask{computer}{beamer-computer-mask}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerimage}{beamer-computer}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerworkingimage}{beamer-computerred}
\pgfdeclareimage[mask=tu,height=.5cm]{logo}{beamer-tu-logo}

\logo{\pgfuseimage{logo}}

\title[Stat 609 Lecture 15]{Stat 609: Mathematical Statistics \\ Lecture 15}
\author[UW-Madison]{Jun Shao}
\institute[Statistics]{%
%  Department of Statistics\\
%  University of Wisconsin\\
%  Madison, WI 53706, USA\\
%~~~\\
  }
\date[2015]{}

\colorlet{redshaded}{red!25!bg}
\colorlet{shaded}{black!25!bg}
\colorlet{shadedshaded}{black!10!bg}
\colorlet{blackshaded}{black!40!bg}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\def\radius{0.96cm}
\def\innerradius{0.85cm}

\def\softness{0.4}
\definecolor{softred}{rgb}{1,\softness,\softness}
\definecolor{softgreen}{rgb}{\softness,1,\softness}
\definecolor{softblue}{rgb}{\softness,\softness,1}

\definecolor{softrg}{rgb}{1,1,\softness}
\definecolor{softrb}{rgb}{1,\softness,1}
\definecolor{softgb}{rgb}{\softness,1,1}

\newcommand{\Bandshaded}[2]{
  \color{shadedshaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\Band}[2]{
  \color{shaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\BaenderNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{input tapes}{}}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{Zwei Eingabebänder}{}}
  \pgfputat{\pgfxy(0,4.25)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \ignorespaces}

\newcommand{\BaenderHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{input tapes}{}}
  \color{shaded}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Bandshaded{}{}}

  \color{blackshaded}
  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{Zwei Eingabebänder}{}}%
  \color{blackshaded}
  \pgfputat{\pgfxy(0,4.25)}{\Bandshaded{}{}}
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}%

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \ignorespaces}

\newcommand{\Slot}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.0)}%
    \pgfbezier{\pgfxy(0.1,3.95)}{\pgfxy(0.05,3.9)}{\pgfxy(0,3.9)}%
    \pgfbezier{\pgfxy(-0.05,3.9)}{\pgfxy(-0.1,3.95)}{\pgfxy(-0.1,4.0)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\SlotZwei}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.25)}%
    \pgfbezier{\pgfxy(0.1,4.25)}{\pgfxy(0.05,4.15)}{\pgfxy(0,4.15)}%
    \pgfbezier{\pgfxy(-0.05,4.15)}{\pgfxy(-0.1,4.2)}{\pgfxy(-0.1,4.25)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}
\def\back{\!\!\!\!\!}
\newcommand{\ClipSlot}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4cm}}}{\pgfxy(0.2,1.5)}\ignorespaces}

\newcommand{\ClipSlotZwei}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4.25cm}}}{\pgfxy(0.2,1.25)}\ignorespaces}


\AtBeginSection[]{\frame{\frametitle{Outline}\tableofcontents[current]}}

\begin{document}

%\frame{\titlepage}

\frame {\vspace{-3mm}
\frametitle{Lecture 15: Multivariate normal distributions}
	\begin{block}{Normal distributions with singular covariance matrices}
		Consider an $n$-dimensional  $X \sim N(\mu, \Sigma) $ with a positive definite $\Sigma$ and a fixed $k\times n$  matrix $A$ that is not of rank $k$
		(so $k$ may be larger than $n$).
		\vspace{1mm}
		
		The mgf of $Y = AX$ is still equal to
		\vspace{-2.5mm}
		\[ M_Y(t) = e^{(A\mu)'t + t' (A \Sigma A') t/2}, \quad  t\in {\cal R}^k  \vspace{-2.5mm} \]
		But what is the distribution corresponding to this mgf?
	\end{block}\vspace{-3mm}
	\begin{alertblock}{Lemma.}
		For any  $n \times n$ non-negative definite matrix  $\Sigma$ and $\mu \in {\cal R}^n$,
		 $e^{\mu 't + t' \Sigma  t/2}$ defined for all $ t\in {\cal R}^n$ is the mgf of an $n$-dimensional random vector $X$.
	\end{alertblock}\vspace{-3mm}
	\begin{alertblock}{Proof.}
		From the theory of linear algebra, a non-negative definite matrix $\Sigma$ of rank $r< n$ satisfies\vspace{-3mm}
		\[ \Sigma   = T' \left( \br{cc}
		\Lambda & 0 \\
		0 & 0 \er \right) T = C' \Lambda C \quad\quad \quad T = \left( \br{c}
		C \\ D \er \right)  \vspace{-3mm} \]
		where
		$\Lambda $ is an $r \times r$ diagonal matrix whose
		all diagonal elements are
	\end{alertblock}}
	\frame{\vspace{-2mm}
		\begin{alertblock}{}
			positive, 0 denotes a matrix of 0's of an appropriate order, $C$ is an $r \times n$ matrix of rank $r$,
			$T$ is an $n \times n$ matrix satisfying $TT' = T'T = I_n $ (the identity matrix of order $n$),
			$CC' = I_r$, $DC' = 0 $, $DD' = I_{n-r}$, and $C'C+D'D = I_n$.
			\vspace{1mm}
			
			Let $Y$ be an $r$-dimensional random vector $\sim N(C \mu , \Lambda )$ and define \vspace{-2mm}
			\[ X = T'  \left( \br{c}
			Y \\
			D\mu  \er \right)  = C'Y + D'D \mu \vspace{-2mm} \]
			Since $Y\sim N(C \mu , \Lambda )$, its mgf is $M_Y(s) = e^{(C\mu)'s + s' \Lambda s/2}$, $s \in {\cal R}^r$ and the mgf of
			$X$ is \vspace{-2mm}
			\beqrs
			M_X(t) \back & = & \back  e^{(D'D\mu)'t} M_Y(Ct)) =
			e^{(D'D\mu)'t} e^{(C\mu)' (Ct) + (Ct)' \Lambda (Ct)/2} \\
			\back & = & \back
			e^{\mu' (D'D+C'C)t + t'C' \Lambda C t/2} = e^{\mu't + t'\Sigma t/2} \quad\quad t \in {\cal R}^n
			\eeqrs
			$~$ \vspace{-7mm}
			
			This completes the proof.
		\end{alertblock}\vspace{-2mm}
		\begin{block}{Definition}
			For any fixed $n \times n$ non-negative definite matrix  $\Sigma$ and $\mu \in {\cal R}^n$, the distribution of
			an $n$-dimensional random vector with mgf $e^{\mu 't + t' \Sigma  t/2}$
			is called normal distribution and denoted by $N(\mu, \Sigma )$.
		\end{block}}
		\frame{
			\begin{alertblock}{}
				\begin{itemize}
					\item
					If $\Sigma $ is positive definition, then this definition is the same as the previous definition using the pdf.
					\item
					If $X \sim N(\mu, \Sigma)$ and $Y = AX + b$,  then $Y \sim N( A\mu , A\Sigma A')$, regardless of whether
					$A\Sigma A'$ is singular or not.
					\item
					If $X$ is multivariate normal, then any sub-vector of $X$ is also normally distributed.
					\item
					If $n$-dimensional $X \sim N(\mu, \Sigma )$ and the rank of $\Sigma$ is $r < n$,
					there exists an $r \times n$ matrix $C$ of rank $r$ and
					$Y= C X \sim N(C\mu , C\Sigma C')$, where $C\Sigma C'$ is a diagonal matrix whose diagonal elements are all positive, and hence
					$Y$ has an $r$-dimensional normal pdf and components of $Y$ are independent.
					\item
					If $n$-dimensional $X \sim N(\mu, \Sigma )$ and the rank of $\Sigma$ is $r < n$, then, from the previous discussion,
					$ X = C'Y + D'D \mu $, where $ Y \sim N(C\mu, C\Sigma C')$ and \vspace{-2mm}
					\beqrs
					E(X ) \back & = & \back C'E(Y) + D'D\mu =( C'C + D'D)\mu = \mu  \\
					{\rm Var}(X) \back & = & \back  C' {\rm Var} (Y) C = C'C \Sigma C'C =  \Sigma
					\eeqrs
				\end{itemize}
			\end{alertblock}
		}
		\frame{\vspace{-2mm}
			\begin{block}{}
				Thus, $\mu$ and $\Sigma$ in $N(\mu , \Sigma )$ is still the mean and covariance matrix.
				\vspace{1mm}
				
				Furthermore, any two components of $X \sim N(\mu , \Sigma )$ are  independent iff they are uncorrelated.
				\vspace{1mm}
				
				This can be shown as follows.
				\vspace{1mm}
				
				Suppose that  $X_1$ and $X_2$ are the first two components of $X$ and Cov$(X_1,X_2) =0$, i.e.,
				the $(1,2)$th and $(2,1)$th elements of $\Sigma$ are 0.
				\vspace{1mm}
				
				Let $\mu_1$ and $\mu_2$ be the first two components of $\mu$ and $\sigma_1^2$ and $\sigma_2^2$ be the first and second diagonal elements of $\Sigma$, and
				let $t = (t_1,t_2,0,...,0)$, $t_1 \in {\cal R}$, $t_2 \in {\cal R}$.
				\vspace{1mm}
				
				Then the mgf of $(X_1,X_2)$ is  \vspace{-2mm}
				\[ M_{(X_1,X_2)}(t_1,t_2) = e^{\mu't + t'\Sigma t/2} = e^{\mu_1t_1  + \sigma_1^2 t_1^2/2}e^{\mu_2 t_2+\sigma_2^2t_2^2 /2}  \quad t_1 \in {\cal R}, \ t_2 \in {\cal R} \vspace{-2mm} \]
				By Theorem M4, $X_1$ and $X_2$ are independent.
			\end{block}
			\vspace{-2mm}
			\begin{alertblock}{Theorem.}
				An $n$-dimensional random vector $X \sim N(\mu, \Sigma )$ (regardless of whether $\Sigma$ is singular or not) iff for any $n$-dimensional constant vector $c$,
				$c' X \sim N(c'\mu , c'\Sigma c)$.
			\end{alertblock}
		}
		
		\frame{
			\begin{alertblock}{Proof.}
				We treat a degenerated $X=c$ as $N( c, 0)$.
				\begin{itemize}
					\item
					If $X \sim N(\mu,\Sigma )$, then $M_X(t) = e^{\mu't + t' \Sigma t/2}$.
					
					For any $c \in {\cal R}^n$, by the properties of mgf, the mgf of $c'X$ is \vspace{-2mm}
					\[ M_{c'X} (t) = M_X(ct) = e^{\mu'(ct) + (ct)' \Sigma (ct)/2} =
					e^{(c'\mu) t + (c'\Sigma c) t^2/2} \quad \quad t \in {\cal R} \vspace{-2mm} \]
					which is the mgf of $N(c'\mu, c'\Sigma c)$.
					
					By uniqueness, $c'X \sim N(c'\mu, c'\Sigma c)$.
					\item
					If $c'X \sim N(c'\mu, c'\Sigma c)$ for any $c \in {\cal R}^n$, then
					$t'X \sim N(t'\mu, t'\Sigma t)$ for any $t \in {\cal R}^n$ and\vspace{-1mm}
					\[ M_{t'X}(s) = e^{(t'\mu )s + (t'\Sigma t)s^2/2} \quad \quad s \in {\cal R} \vspace{-1mm} \]
					Letting $s= 1$, we obtain \vspace{-1mm}
					\[ M_{t'X}(1) = e^{(t'\mu ) + (t'\Sigma t)/2} = E(e^{t'X}) = M_X(t) \quad \quad t \in {\cal R}^n  \vspace{-1mm} \]
					By uniqueness, $X \sim N(\mu,\Sigma )$.
				\end{itemize}
				The condition {\bf any} $c \in {\cal R}^n$ is important.
			\end{alertblock}
		}
		\frame{
			\begin{block}{The uniform distribution on $ [a, b] \times [c,d]$}
				We have shown that the two marginal distributions are uniform distributions on intervals $[a,b]$ and $[c,d]$.
				
				For non-zero constants $\xi $ and $\zeta$, is the distribution of $\xi X + \zeta Y$ a uniform distribution on some interval?
				
				If $(e^{bt}-e^{at})/t$ is defined to be $b-a$ when $t=0$ for any constants $a < b$, then
				\vspace{-2mm}
				\beqrs
				M_{X,Y}(t,s) &=& \int_a^b \int_c^d e^{tx+sy} \frac{1}{(b-a)(d-c)} dx dy \\
				&=& \frac{(e^{bt}-e^{at})(e^{ds}-e^{cs})}{(b-a)(d-c)ts} \quad \quad s, t \in {\cal R} \vspace{-2mm}
				\eeqrs
				and\vspace{-2mm}
				\[ M_{\xi X+ \zeta Y}(t) = E(e^{t(\xi X + \zeta Y)}) = \frac{(e^{b\xi t}-e^{a\xi t})(e^{d \zeta t}-e^{c \zeta t})}{(b-a)(d-c)\xi \zeta t^2} \quad \quad t \in {\cal R}
				\vspace{-2mm} \]
				This is not a mgf of a uniform distribution on an interval $[r,h]$, which is of the form
				$(e^{ht}-e^{rt})/[t(h-r)]$ for $t \in {\cal R}$.
			\end{block}
		}
\frame{
\begin{alertblock}{}\vspace{-1mm}
We have shown that if $X \sim N(\mu, \Sigma )$, then any linear function $AX+b$ is normally distributed.
\vspace{1mm}

The following result concerns the independence of linear functions of a normally distributed random vector.
\end{alertblock}\vspace{-2mm}
\begin{block}{Theorem N1.}
Let $X$ be an $n$-dimensional random vector $\sim N(\mu , \Sigma)$ and
$A$ be a fixed $k \times n$ matrix, and $B$ be a fixed $l\times n$ matrix.
Then, $AX$ and $BX$ are independent iff $A \Sigma B' =0$.
\end{block}\vspace{-2mm}
\begin{block}{Proof.}
Let \vspace{-2mm}
\[ Y = \left( \br{c}
A \\
B \er \right) X =  \left( \br{c}
AX \\
BX \er \right) \vspace{-2mm}
\]
From the properties of the multivariate normal distribution, we know that $Y$ is multivariate normal with covariance matrix
\vspace{-2mm}
\[ \left( \br{c}
A \\
B \er \right) \Sigma ( A' \ \, B' ) = \left( \br{cc}
A\Sigma A' & A\Sigma B' \\
B\Sigma A' & B\Sigma B' \er \right) \]
\end{block}}
\frame{\vspace{-2mm}
\begin{block}{}
Hence, $AX$ and $BX$ are uncorrelated iff $A\Sigma B' = 0 $ and, thus, the only if part follows since independence implies no correlation.
\vspace{1mm}

The proof for the if part is the same as the proof of two uncorrelated components of $X$ are independent:
we can show that if $A\Sigma B' =0$, then the mgf of $(AX,BX)$
is a product of an mgf on ${\cal R}^k$  and another mgf on ${\cal R}^l$, and then apply Theorem M4.
\end{block}\vspace{-2mm}
\begin{alertblock}{Theorem N2.}
If $(X,Y)$ is a random vector  $\sim N(\mu, \Sigma )$ with \vspace{-3mm}
\[ \mu = \left( \br{cc}
\mu_1 \\
\mu_2 \er \right), \quad\quad \quad \Sigma = \left( \br{cc}
\Sigma_{11} & \Sigma_{12} \\
\Sigma_{21} & \Sigma_{22} \er \right) ,\vspace{-3mm} \]
and if $\Sigma$ is positive definite, then \vspace{-3mm}
\[ Y|X \sim N\left( \mu_2 +  \Sigma_{21} \Sigma_{11}^{-1} (X-\mu_1)  ,
\Sigma_{22}-\Sigma_{21}\Sigma^{-1}_{11}\Sigma_{12}\right) \vspace{-0.5mm} \]
\end{alertblock}
\vspace{-2mm}
\begin{alertblock}{}
It follows from the properties of normal distributions that \vspace{-2.5mm}
\[ E(Y|X) =  \mu_2 +  \Sigma_{21} \Sigma_{11}^{-1} (X-\mu_1), \quad \quad  {\rm Var}(Y|X) = \Sigma_{22}-\Sigma_{21}\Sigma^{-1}_{11}\Sigma_{12} \vspace{-2.5mm}
 \]
While the conditional mean depends on $X$, the conditional covariance matrix does not.
\end{alertblock}}
\frame{
\begin{alertblock}{Proof.}
Consider the transformation \vspace{-2mm}
\[ U = A X + Y \vspace{-2mm} \]
with a fixed matrix $A$ chosen so that $U$ and $X$ are independent.
\vspace{1mm}

From Theorem N1, we need $U$ and $X$ to be uncorrelated. \vspace{1mm}

Since \vspace{-2mm}
\beqrs
 {\rm Cov}(X,U)  \back & = & \back {\rm Cov}(X, AX+Y ) = {\rm Cov}(X,AX) + {\rm Cov}(X,Y) \\
 \back & = & \back {\rm Cov}(X,X)A'  + \Sigma_{12} = \Sigma_{11}A' + \Sigma_{12}
 \eeqrs
 $~$ \vspace{-6mm}

 we choose $A = - \Sigma_{21} \Sigma_{11}^{-1} $.
\vspace{1mm}

Consider the transformation \vspace{-2mm}
\[ \left( \br{c}
V \\
U \er \right) = \left( \br{c}
X \\
AX+Y \er \right) =
\left( \br{cc}
I & 0  \\
-\Sigma_{21}\Sigma_{11}^{-1} & I \er  \right)
\left( \br{c}
X \\
Y \er \right), \quad\quad  \bigg| \frac{\partial (U,V) }{\partial (X,Y) } \bigg| = 1 \vspace{-2mm} \]
Let $f_{(X,Y})$ be the pdf of $(X,Y)$, $f_{(U,V)}$ be the pdf of $(U,V)$, $f_U$ be the pdf of $U$ and $f_V$ be the pdf of $V$.
\vspace{1mm}

By the transformation formula and the independence of $U$ and $V=X$,
\end{alertblock}
}
\frame{
\begin{alertblock}{}
\[ f_{(X,Y)} (x,y) = f_{(U,V)}(u,v) = f_U(u)f_V(v) = f_U(y-\Sigma_{21}\Sigma_{11}^{-1}x )f_X(x)  \]
Then the pdf of $Y|X$ is \vspace{-2mm}
\[ \frac{f_{(X,Y)}(x,y)}{f_X(x)} = \frac{ f_U(y-\Sigma_{21}\Sigma_{11}^{-1}x )f_X(x) }{f_X(x)}
= f_U(y-\Sigma_{21}\Sigma_{11}^{-1}x ) \vspace{-2mm} \]
Since $U = - \Sigma_{21}\Sigma_{11}^{-1} X+Y$, $U$ is normally distributed. \vspace{-2mm}
\[ E(U) = - \Sigma_{21}\Sigma_{11}^{-1}E(X) +E(Y) =
- \Sigma_{21}\Sigma_{11}^{-1}\mu_1 +\mu_2  \vspace{-4mm} \]
\beqrs
{\rm Var}(U) \back & = & \back {\rm Var}(AX+Y) = {\rm Var}(AX) + {\rm Var}(Y) +2 {\rm Cov}(AX,Y) \\
\back & = & \back A {\rm Var}(X)A' + \Sigma_{22} + 2A {\rm Cov}(X,Y) \\
\back & = & \back \Sigma_{21}\Sigma_{11}^{-1} \Sigma_{11} \Sigma_{11}^{-1} \Sigma_{12} + \Sigma_{22} - 2\Sigma_{21}\Sigma_{11}^{-1} \Sigma_{12} \\
\back & = & \back  \Sigma_{22} - \Sigma_{21}\Sigma_{11}^{-1} \Sigma_{12}
\eeqrs
$~$ \vspace{-7mm}

Hence, $f_U$ is the pdf of $N(\mu_2 - \Sigma_{21}\Sigma_{11}^{-1}\mu_1,  \Sigma_{22} - \Sigma_{21}\Sigma_{11}^{-1} \Sigma_{12})$.
  \vspace{1mm}

  Given $X=x$, $\Sigma_{21}\Sigma_{11}^{-1}x $ is a constant and, hence, $f_U(y-\Sigma_{21}\Sigma_{11}^{-1}x )$
  is the pdf of $N(\mu_2 + \Sigma_{21}\Sigma_{11}^{-1}(x-\mu_1),  \Sigma_{22} - \Sigma_{21}\Sigma_{11}^{-1} \Sigma_{12})$, considered as a function of $y$. \end{alertblock}}
  \frame{\vspace{-2mm}
  \begin{block}{Quadratic forms}
  For a random vector $X$ and a fixed symmetric matrix $A$, $X'AX$ is called a quadratic function
  or quadratic form of $X$.
  \vspace{1mm}

  We now study the distribution of quadratic forms when $X$ is multivariate normal.
  \end{block}\vspace{-2mm}
  \begin{alertblock}{Theorem N3.}
  Let $X \sim N(\mu , I_n)$ and $A$ be a fixed $n \times n$ symmetric matrix.
  A necessary and sufficient condition for $X'AX  $ is chi-square distributed is $A^2 =A$, in which case the degrees of freedom of the chi-square distribution is the rank of $A$ and the noncentrality parameter $\mu' A \mu$.
  \end{alertblock}\vspace{-2mm}
  \begin{alertblock}{Proof.}
  Sufficiency.

  If $A^2 = A$, then $A$ is a projection matrix and there exists an $n\times n$ matrix $T$ such that $T'T=TT' = I_n$ and \vspace{-3mm}
  \[ A = T' \left( \br{cc}
  I_k & 0 \\
  0 & 0 \er \right) T = C'C  \vspace{-3mm} \]
  where $k$ is the rank of $A$ and $C$ is the first $k$ rows of $T$.
  \end{alertblock}
  }
  \frame{\vspace{-1.5mm}
  \begin{alertblock}{}
Then $ X'AX = (CX)'(CX)$ is simply the sum of the squares of $CX$, the first $k$ components of $TX$.
\vspace{1mm}

Since $TX \sim N(T\mu , TI_nT') = N(T\mu, I_n)$, by definition $X'AX$ has the chi-square distribution with degrees of freedom $k$ and
noncentrality parameter $(C\mu)'(C \mu) = \mu' C' C\mu = \mu' A \mu$.
\vspace{1.5mm}

Necessity.

Suppose that $X'AX$ is chi-square with degrees of freedom $m$ and noncentrality parameter $\delta \geq 0$. 

Then $A$ must be nonnegative definite and  there exists an $n\times n$ matrix $T$ such that $T'T= $  $TT' = I_n$ and \vspace{-3mm}
  \[ A = T' \left( \br{cc}
  \Lambda & 0 \\
  0 & 0 \er \right) T  \vspace{-3mm} \]
where $\Lambda$ is a $k\times k$ diagonal matrix contains
$k$ non-zero eigenvalues $0< \lambda_1 \leq \cdots \leq \lambda_k$.

We still have $TX \sim N(T\mu , I_n)$.

Let $Y_1,...,Y_k$ be the first $k$ components of $TX$.

Then $Y_i^2$'s are independent and $Y_i^2 \sim $ chi-square with degree of
freedom 1 and noncentrality parameter
$\mu_i^2$, where $\mu_i$ is the $i$th
\end{alertblock}}
\frame{\vspace{-2mm}
\begin{alertblock}{}
 component of $\mu$, and 
\vspace{-1mm}
\[ X'AX = \sum_{i=1}^k \lambda_i Y_i^2 \vspace{-2mm} \]
Using the mgf formula for noncentral chi-square distributions, the mgf's of the left and right hand sides are respectively 
 given in the left and right hand sides of the following: \vspace{-2mm}
\[ 
 \frac{e^{\delta t/(1-2t)}}{(1-2t)^{m/2}} = \prod_{i=1}^k \frac{e^{\lambda_i \mu_i^2 t / (1-2\lambda_i t)}}{(1-2\lambda_i t )^{1/2}} \quad 
\quad t < 1/2 \vspace{-2mm} 
\]
Suppose that $\lambda_k > 1$. 

When $t \rightarrow (2 \lambda_k)^{-1}$, the right hand side of the above equation diverges to $\infty$ whereas the left hand side of the above equation goes to
$ e^{\delta (2\lambda_k)^{-1} /(1-\lambda_k^{-1})}/ (1-\lambda_k^{-1})^{m/2}  < \infty$, which is a contradiction. 

Hence $\lambda_k \leq 1$ so that $\lambda_i \leq 1$ for all $i$. 

Suppose that $\lambda_k =\cdots = \lambda_{l+1}=1 > \lambda_l \geq \cdots \geq \lambda_1 >0$ for a positive integer $l\leq k$, which implies 
 \vspace{-2mm}
\[
 \frac{e^{\delta t/(1-2t)}}{(1-2t)^{(m-k+l)/2}} = \prod_{i=1}^l \frac{e^{\lambda_i \mu_i^2 t / (1-2\lambda_i t)}}{(1-2\lambda_i t )^{1/2}} \quad
\quad t < 1/2 
\] 
  \end{alertblock}
  }
\frame{
\begin{alertblock}{}
When $t \rightarrow 1/2$, the left hand side of the above equation diverges to $\infty$, whereas the right hand side of the above equation converges 
to \vspace{-2mm}
\[ \prod_{i=1}^l \frac{e^{\lambda_i \mu_i^2  / 2(1-\lambda_i )}}{(1-\lambda_i  )^{1/2}} \vspace{-2mm} \]
which is a contradiction. 

Therefore, we must have $\lambda_1 = \cdots = \lambda_k = 1$, i.e., $A$ is a projection matrix. 
\end{alertblock}\vspace{-2mm}
\begin{block}{Theorem N4 (Cochran's theorem).}
Suppose that $X$ is an $n$-dimensional random vector $ \sim N(\mu, I_n)$ and\vspace{-2mm} 
\[ X' X = X' A_1X + \cdots + X' A_kX , \vspace{-2mm} \]
where $I_n$ is the $n\times n$ identity matrix
and $A_i$ is an $n \times n$ symmetric matrix with rank $n_i$,
$i=1,...,k$. A necessary and sufficient condition for 
\begin{itemize}
\item[(i)] 
$X' A_iX$
has the noncentral chi-square distribution with degrees of freedom $n_i$ and noncentrality parameter $ \delta_i $,
$i=1,...,k$,
\item[(ii)]
 $X' A_iX$'s are independent,
 \end{itemize}
  is $n=
n_1+ \cdots +n_k$, in which case $\delta_i = \mu' A_i \mu$ and
$\delta_1+ \cdots + \delta_k = \mu' \mu$.
\end{block}}
\frame{\vspace{-2mm}
\begin{block}{Proof.}
Suppose that (i)-(ii) hold. \vspace{1mm}

Then $X' X$ has the chi-square distribution with degrees of freedom $n_1+\cdots + n_k$ and noncentrality parameter 
$ \delta_1+\cdots + \delta_k$.\vspace{1mm}

 By definition, $X' X$ has the
noncentral chi-square distribution with degrees of freedom $n$ and noncentrality parameter $ \mu' \mu $.
\vspace{1mm}

Then we must have  $n=n_1+ \cdots +n_k$ and
$\delta_1+ \cdots + \delta_k = \mu' \mu$.
\vspace{1mm}

Suppose now that $n=n_1+\cdots + n_k$.
\vspace{1mm}

 From the theory of linear algebra, for each $i$ there exists $c_{ij} \in {\cal R}^n$,
$j=1,...,n_{i}$, such that\vspace{-2mm} 
\[  X' A_iX = \pm ( c'_{i1}X )^2 \pm \cdots \pm (c'_{in_i}X)^2
\vspace{-2mm} \]
Let $C$ be the $n \times n$ matrix whose columns are $c_{11},...,c_{1n_1},...,c_{k1},...,c_{kn_k}$,

Then\vspace{-1mm} 
\[ X' X = X' C \Delta C' X \vspace{-2mm} \]
with an $n\times n$ diagonal matrix
$\Delta$ whose diagonal elements are  $\pm 1$. 
\vspace{1mm}

This implies
$C \Delta C' = I_n$ and thus $C$ is of full rank and $\Delta
= C^{-1}(C')^{-1}$, 
which is positive definite.
\end{block}}
\frame{\vspace{-2mm} 
\begin{block}{}
This shows
$\Delta = I_n$, which implies $C' C =CC' = I_n$ and\vspace{-2mm} 
\[ X' A_iX = \sum_{j=n_1+\cdots + n_{i-1}+1}^{n_1+\cdots + n_{i-1}+n_i}
Y_j^2 , \vspace{-2mm} \]
where $Y_j$ is the $j$th component of $Y=C 'X \sim N( C' \mu , I_n)$. 
\vspace{1mm}

Hence $Y_j$'s are independent and
$Y_j \sim N(\lambda_j,1)$, where $\lambda_j$ is the $j$th component of $C' \mu$.
\vspace{1mm}

This shows that $X' A_i X$, $i=1,...,k$, are independent and 
$X' A_iX$ has the chi-square distribution with degrees of freedom 
$n_i$ and noncentrality parameter $\delta_i = \lambda_{n_1+\cdots + n_{i-1}+1}^2 + \cdots +
\lambda_{n_1+\cdots + n_{i-1}+n_i}^2 $.
\vspace{1mm}

Letting $X=\mu$ and $Y=C'X = C'\mu$, we obtain that
$\delta_i = \mu' A_i\mu$ and
$\delta_1+ \cdots + \delta_k = \mu' C C' \mu = \mu' \mu$.
\vspace{1mm}

This completes the proof. 
\end{block}\vspace{-2mm}
\begin{alertblock}{Theorem N5.}
Let $X$ be an $n$-dimensional random vector $ \sim N(\mu, I_n)$ and 
$A_1$ and $A_2$ be $n \times n$ projection matrices.
Then a necessary and sufficient condition that $X' A_1X$ and
$X' A_2X$ are independent is $A_1A_2 =0$.
\end{alertblock}
}
\frame{
\begin{alertblock}{Proof.}
If $A_1A_2 = 0$, then \vspace{-2mm} 
\beqrs
(I_n-A_1-A_2)^2 \back & = & \back I_n-A_1-A_2 -A_1+A_1^2 +A_2A_1
-A_2+A_1A_2+A_2^2 \\
&  = & \back I_n - A_1-A_2 ,
\eeqrs
$~$ \vspace{-7mm}

i.e., $I_n-A_1-A_2$ is a projection matrix
with rank $= \mbox{trace}(I_n-A_1-A_2) $ $= n - r_1-r_2$, where
$r_i = \mbox{trace}(A_i)$ is the rank of $A_i$, $i=1,2$. 
\vspace{1mm}

By Cochran's theorem
and\vspace{-2mm} 
\[ X' X = X' A_1X + X' A_2X
+ X' (I_n-A_1-A_2)X, \vspace{-2mm} \]
$X' A_1X$ and $X' A_2X$ are independent.
\vspace{1mm}

This proves the sufficiency. 
\vspace{1mm}

Assume that $X' A_1X$ and $X' A_2X$ are independent.
\vspace{1mm}

Since $X' A_iX$ has the noncentral chi-square distribution with degrees of freedom 
$r_i = $ the rank of $A_i$ and noncentrality parameter 
$\delta_i = \mu'  A_i\mu$, $X' (A_1+A_2)X$ has the noncentral
chi-square distribution with degrees of freedom $r_1+r_2$ and noncentrality parameter  $\delta_1+\delta_2$.
\end{alertblock}}
\frame{
\begin{alertblock}{}
Consequently, $A_1+A_2$ is a projection matrix, i.e., \vspace{-2mm} 
\[ (A_1+A_2)^2 = A_1+A_2 , \vspace{-2mm} \]
which implies\vspace{-2mm} 
\[ A_1A_2+A_2A_1 = 0 .\vspace{-2mm} \]
Since $A_1^2=A_1$, we obtain that\vspace{-2mm} 
\[ 0 = A_1(A_1A_2+A_2A_1 ) = A_1A_2 + A_1A_2A_1\vspace{-2mm}  \]
and\vspace{-2mm} 
\[ 0 = A_1(A_1A_2+A_2A_1 )A_1 = 2 A_1A_2A_1 , \vspace{-2mm} \]
which imply $A_1A_2=0$.
\vspace{1mm}

This proves the necessity.  
\end{alertblock}}
\end{document}


