% $Header: /cvsroot/latex-beamer/latex-beamer/examples/beamerexample5.tex,v 1.22 2004/10/08 14:02:33 tantau Exp $

\documentclass[11pt]{beamer}
\usepackage{amscd}

%\usetheme{Madrid} % Darmstadt, Boadilla


\mode<presentation> {
  \usetheme{Madrid}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsbsy,alltt,amsmath,times}
\usepackage{epsfig, color, graphicx, rotate, colordvi}
%\usepackage{fancybox,fancyhdr,times,subfigure}
\usepackage{hyperref, mathptmx}


\setbeamercovered{dynamic}

\newcommand{\Lang}[1]{\operatorname{\text{\textsc{#1}}}}

\newcommand{\Class}[1]{\operatorname{\mathchoice
  {\text{\sf \small #1}}
  {\text{\sf \small #1}}
  {\text{\sf #1}}
  {\text{\sf #1}}}}

\newcommand{\NumSAT}      {\text{\small\#SAT}}
\newcommand{\NumA}        {\#_{\!A}}

\newcommand{\barA}        {\,\bar{\!A}}

\newcommand{\Nat}{\mathbb{N}}
\newcommand{\Set}[1]{\{#1\}}

\newtheorem{thm}{Result}%[section]
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{defn}[thm]{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{rem}{Remark}

\newcommand{\cred}{\textcolor {red}}
\newcommand{\cblue}{\textcolor {blue}}
\newcommand{\cgreen}{\textcolor {green}}

\def\ul{\underline}

\def\sign{\;{\rm sign}}
\def\rank{\;{\rm rank}}
\def\disp{\displaystyle}
\def\sn{\sqrt{n}}
\def\Pr{\mathrm{P}}

\def\Cov{\;{\rm Cov}}
\def\L{\underline L}
\def\LL{\overline L}

\def\balpha{{\boldsymbol{\alpha}}}
\def\cal{\mathcal}
\def\conas{\stackrel{a.s.} {\longrightarrow}}         % conv. a.s.
\def\conp{\stackrel{\cal P} {\longrightarrow}}        % conv. in probability
\def\conD{\stackrel{\mathcal{L}} {\longrightarrow}}        % conv. in distribution
\def\conw{\stackrel{\mathcal{L}} {\rightarrow}}        % conv. in distribution
\def\hat{\widehat}
%\def\endproof{\begin{flushright}$\lozenge$\end{flushright}}
\def\Proof{\noindent{\it Proof }}
\def\endproof{\hfill{$\Box$}}
\newcommand{\wcon}{\stackrel{\mathcal{L}}{\longrightarrow}}
\def\Var{\mbox{var}}
\def\br{\begin{array}}
\def\er{\end{array}}

%%%%%%%%%%%%%%% MY DEFINITIONS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Put your own definitions
%
\def\VaR{{\rm VaR}}
\def\E{{\rm E}}
\def\beqrs{\begin{eqnarray*}}           %\beqrs
\def\eeqrs{\end{eqnarray*}}             %\eeqrs

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\pgfdeclaremask{tu}{beamer-tu-logo-mask}
\pgfdeclaremask{computer}{beamer-computer-mask}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerimage}{beamer-computer}
\pgfdeclareimage[interpolate=true,mask=computer,height=2cm]{computerworkingimage}{beamer-computerred}
\pgfdeclareimage[mask=tu,height=.5cm]{logo}{beamer-tu-logo}

\logo{\pgfuseimage{logo}}

\title[Stat 609 Lecture 11]{Stat 609: Mathematical Statistics \\ Lecture 11}
\author[UW-Madison]{Jun Shao}
\institute[Statistics]{%
%  Department of Statistics\\
%  University of Wisconsin\\
%  Madison, WI 53706, USA\\
%~~~\\
  }
\date[2015]{}

\colorlet{redshaded}{red!25!bg}
\colorlet{shaded}{black!25!bg}
\colorlet{shadedshaded}{black!10!bg}
\colorlet{blackshaded}{black!40!bg}

\colorlet{darkred}{red!80!black}
\colorlet{darkblue}{blue!80!black}
\colorlet{darkgreen}{green!80!black}

\def\radius{0.96cm}
\def\innerradius{0.85cm}

\def\softness{0.4}
\definecolor{softred}{rgb}{1,\softness,\softness}
\definecolor{softgreen}{rgb}{\softness,1,\softness}
\definecolor{softblue}{rgb}{\softness,\softness,1}

\definecolor{softrg}{rgb}{1,1,\softness}
\definecolor{softrb}{rgb}{1,\softness,1}
\definecolor{softgb}{rgb}{\softness,1,1}

\newcommand{\Bandshaded}[2]{
  \color{shadedshaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\Band}[2]{
  \color{shaded}
  \pgfmoveto{\pgfxy(-0.5,0)}
  \pgflineto{\pgfxy(-0.6,0.1)}
  \pgflineto{\pgfxy(-0.4,0.2)}
  \pgflineto{\pgfxy(-0.6,0.3)}
  \pgflineto{\pgfxy(-0.4,0.4)}
  \pgflineto{\pgfxy(-0.5,0.5)}
  \pgflineto{\pgfxy(4,0.5)}
  \pgflineto{\pgfxy(4.1,0.4)}
  \pgflineto{\pgfxy(3.9,0.3)}
  \pgflineto{\pgfxy(4.1,0.2)}
  \pgflineto{\pgfxy(3.9,0.1)}
  \pgflineto{\pgfxy(4,0)}
  \pgfclosepath
  \pgffill

  \color{black}
  \pgfputat{\pgfxy(0,0.7)}{\pgfbox[left,base]{#1}}
  \pgfputat{\pgfxy(0,-0.1)}{\pgfbox[left,top]{#2}}
}

\newcommand{\BaenderNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{input tapes}{}}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiNormal}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Band{Zwei Eingabebänder}{}}
  \pgfputat{\pgfxy(0,4.25)}{\Band{}{}}

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \ignorespaces}

\newcommand{\BaenderHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{input tapes}{}}
  \color{shaded}
  \pgfputat{\pgfxy(0.35,4.6)}{\pgfbox[center,base]{$\vdots$}}
  \pgfputat{\pgfxy(0,4)}{\Bandshaded{}{}}

  \color{blackshaded}
  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$w_1$}}

  \pgfxyline(0,4)(0,4.5)
  \pgfxyline(1.8,4)(1.8,4.5)
  \pgfputat{\pgfxy(0.25,4.25)}{\pgfbox[left,center]{$w_n$}}
  \ignorespaces}

\newcommand{\BaenderZweiHell}
{%
  \pgfsetlinewidth{0.4pt}
  \color{black}
  \pgfputat{\pgfxy(0,5)}{\Bandshaded{Zwei Eingabebänder}{}}%
  \color{blackshaded}
  \pgfputat{\pgfxy(0,4.25)}{\Bandshaded{}{}}
  \pgfputat{\pgfxy(0.25,4.5)}{\pgfbox[left,center]{$v$}}
  \pgfputat{\pgfxy(0.25,5.25)}{\pgfbox[left,center]{$u$}}%

  \pgfxyline(0,5)(0,5.5)
  \pgfxyline(1.2,5)(1.2,5.5)

  \pgfxyline(0,4.25)(0,4.75)
  \pgfxyline(1.8,4.25)(1.8,4.75)
  \ignorespaces}

\newcommand{\Slot}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.0)}%
    \pgfbezier{\pgfxy(0.1,3.95)}{\pgfxy(0.05,3.9)}{\pgfxy(0,3.9)}%
    \pgfbezier{\pgfxy(-0.05,3.9)}{\pgfxy(-0.1,3.95)}{\pgfxy(-0.1,4.0)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\SlotZwei}[1]{%
  \begin{pgftranslate}{\pgfpoint{#1}{0pt}}%
    \pgfsetlinewidth{0.6pt}%
    \color{structure}%
    \pgfmoveto{\pgfxy(-0.1,5.5)}%
    \pgfbezier{\pgfxy(-0.1,5.55)}{\pgfxy(-0.05,5.6)}{\pgfxy(0,5.6)}%
    \pgfbezier{\pgfxy(0.05,5.6)}{\pgfxy(0.1,5.55)}{\pgfxy(0.1,5.5)}%
    \pgflineto{\pgfxy(0.1,4.25)}%
    \pgfbezier{\pgfxy(0.1,4.25)}{\pgfxy(0.05,4.15)}{\pgfxy(0,4.15)}%
    \pgfbezier{\pgfxy(-0.05,4.15)}{\pgfxy(-0.1,4.2)}{\pgfxy(-0.1,4.25)}%
    \pgfclosepath%
    \pgfstroke%
  \end{pgftranslate}\ignorespaces}

\newcommand{\ClipSlot}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4cm}}}{\pgfxy(0.2,1.5)}\ignorespaces}

\newcommand{\ClipSlotZwei}[1]{%
  \pgfrect[clip]{\pgfrelative{\pgfxy(-0.1,0)}{\pgfpoint{#1}{4.25cm}}}{\pgfxy(0.2,1.25)}\ignorespaces}


\AtBeginSection[]{\frame{\frametitle{Outline}\tableofcontents[current]}}

\begin{document}
\def\back{\!\!\!\!\!}

%\frame{\titlepage}

\frame {
\frametitle{Lecture 11: Correlation and independence}
\vspace{-2mm}
\begin{alertblock}{Definition 4.5.1.}
	The covariance of random variables $X$ and $Y$ is defined as\vspace{-2mm}
	\[ {\rm Cov}(X,Y) = E[(X-E(X)][Y-E(Y)] = E(XY) -E(X)E(Y)\vspace{-2mm}\]
	provided that the expectation exists.
	\end{alertblock}\vspace{-2mm}
	\begin{alertblock}{Definition 4.5.2.}
		The correlation (coefficient) of random variables $X$ and $Y$ is defined as\vspace{-2mm}
		\[ \rho_{X,Y} = \frac{{\rm Cov}(X,Y)}{\sqrt{{\rm Var}(X){\rm Var}(Y)}} \]
		\end{alertblock}
		\vspace{-2mm}
			\begin{block}{}
				\begin{itemize}
					\item
					By Cauchy-Schwartz's inequality, $[{\rm Cov}(X,Y)]^2 \leq {\rm Var}(X) {\rm Var}(Y)$ and, hence,
					$| \rho_{X,Y}| \leq 1$.
								\item
								If large values of $X$ tend to be observed with large (or small) values of $Y$ and small values of $X$ with small (or large) values of $Y$, then
								Cov$(X,Y)>0$ (or $<0$). \end{itemize}
					\end{block}}
					\frame{\vspace{-2.5mm}
						\begin{block}{}
							\begin{itemize}
								\item
								If Cov$(X,Y)=0$, then we say that $X$ and $Y$ are uncorrelated.\vspace{-0.5mm}
								\item
								The correlation is a standardized value of the covariance.
								\end{itemize}
								\end{block}\vspace{-2.5mm}
								\begin{alertblock}{Theorem 4.5.6.}
									If $X$ and $Y$ are random variables and $a$ and $b$ are constants, then \vspace{-3mm}
									\[ {\rm Var}(aX+bY) = a^2 {\rm Var}(X) + b^2 {\rm Var}(Y)+2ab {\rm Cov}(X,Y) \]
									\end{alertblock}\vspace{-3mm}
									\begin{alertblock}{}
										Theorem 4.5.6 with $a=b=1$ implies that, if $X$ and $Y$ are positively correlated, then the variation in $X+Y$ is greater than the sum of the variations in $X$ and $Y$; but if they are negatively correlated, then the variation in $X+Y$ is less than the sum of the variations.
										\vspace{1mm}
										
										This result is useful in statistical applications.\vspace{-0.5mm}
										\end{alertblock}\vspace{-2.5mm}
										\begin{block}{Multivariate expectation}
											The expectation of a random vector $X=(X_1,...,X_n)$ is defined
											as $E(X) = ( E(X_1),...,E(X_n))$, provided that $E(X_i)$ exists for any $i$.
											\vspace{1mm}
											
											When $M$ is a matrix whose $(i,j)$th element if a random variable $X_{ij}$,
											$E(M)$ is defined as the matrix whose $(i,j)$th element is $E(X_{ij})$,
											provided that $E(X_{ij})$ exists for any $(i,j)$.
										\end{block}}
										\frame{\vspace{-2mm}
										\begin{alertblock}{Variance-covariance matrix}
											The concept of mean and variance can be extended to random vectors:
											for an $n$-dimensional  random vector $X=(X_1,...,X_n)$, its mean is $E(X)$ and its variance-covariance matrix is  \vspace{-2mm}
											\[ 
											{\rm Var}(X) =  E\{ [X-E(X)][X-E(X)]'\}
											 =  E(XX') - E(X)E(X')
\vspace{-2mm}											\]
											which is an $n\times n$ symmetric matrix whose $i$th diagonal element is
											the variance  ${\rm Var}(X_i)$ and $(i,j)$th off-diagonal element is the covariance ${\rm Cov}(X_i,X_j)$.
											
											${\rm Var}(X)$ is nonnegative definite.
											
							If the rank of ${\rm Var}(X)$ is $r<n$, then, with probability equal to 1, $X$ is in a subspace of ${\cal R}^n$ with dimension $r$.
											
											If $A$ is a constant $m\times n$ matrix, then \vspace{-3mm}
											\[ E(AX) = AE(X) \quad \quad \mbox{and} \quad \quad {\rm Var}(AX) = A{\rm Var}(X) A' \]
												
										\end{alertblock}
								 			\vspace{-2mm}
										\begin{block}{Example 4.5.4.}
											The joint pdf of $(X,Y)$ is \vspace{-3mm}
											\[ f(x,y) = \left\{ \br{ll}
											1 & \quad 0<x< 1, \ x<y<x+1 \\
											0 & \quad \mbox{otherwise} \er \right.  \]
											\end{block}
}
		\frame{\vspace{-2mm}
			\begin{block}{}
				The marginal of $X$ is $uniform(0,1)$, since for $0<x<1$, \vspace{-2.5mm}
				\[ f_X(x) = \int_{-\infty}^\infty f(x,y) dy = \int_x^{x+1} dy = x+1-x = 1 \vspace{-2.5mm} \]
				For $0<y<1$, \vspace{-3mm}
				\[  f_Y(y) = \int_{-\infty}^\infty f(x,y) dx = \int_0^{y} dx = y - 0 = y \vspace{-3mm} \]
				and for $1 \leq y < 2$, \vspace{-3mm}
				\[  f_Y(y) = \int_{-\infty}^\infty f(x,y) dx = \int_{y-1}^1 dx = 1- (y-1)=2 - y \vspace{-2mm} \]
				i.e., \vspace{-2mm}
				\[ f_Y(y) = \left\{ \br{ll}
				2-y & \quad 1 \leq  y < 2 \\
				y & \quad 0< y< 1 \\
				0 & \quad \mbox{otherwise} \er \right.  \vspace{-1mm} \]
				Thus, $E(X)=1/2$ and Var$(X) = 1/12$, and \vspace{-3mm}
				\[ E(Y) = \int_0^1 y^2 dy + \int_1^2 y(2-y) dy = \frac{1}{3} + 4 -1 - \frac{1}{3} (8-1) =1 \vspace{-2mm} \]
				\[ {\rm Var} (Y) = E(Y^2) - 1 \!=\! \int_0^1 y^3 dy + \! \int_1^2 y^2(2-y) dy - 1
				= \frac{1}{4} + \frac{14}{3} - \frac{15}{4} -1 = \frac{1}{6}  \]
			\end{block}}
			\frame{\vspace{-2mm}
				\begin{block}{}
					Also, \vspace{-2mm}
					\[ E(XY) = \int_0^1 \int_x^{x+1}xy dy dx = \int_0^1 \left( x^2+ \frac{x}{2} \right) dx = \frac{1}{3} + \frac{1}{4} =\frac{7}{12}\vspace{-2mm} \]
					Hence, \vspace{-4mm}
					\[ {\rm Cov}(X,Y) = E(XY) - E(X)E(Y) = \frac{7}{12} - \frac{1}{2} = \frac{1}{12} \vspace{-2mm} \]
					\[ \rho_{X,Y} = \frac{{\rm Cov}(X,Y) }{\sqrt{{\rm Var}(X){\rm Var}(Y)}}
					= \frac{1/12}{\sqrt{1/12 \times 1/6}}\frac{1}{\sqrt{2}} \]
				\end{block}
				\vspace{-2.5mm}
				\begin{alertblock}{Theorem 4.5.7.}
					For random variables $X$ and $Y$, $|\rho_{X,Y}|=1$ iff
					$P(Y= aX+b)=1$ for constants $a $ and $b$, where $a>0$ if $\rho_{X,Y} =1$ and $a<0$ if $\rho_{X,Y} = -1$.
				\end{alertblock}
				\vspace{-2.5mm}
				\begin{block}{}
					\begin{itemize}
						\item
						The proof of this theorem is actually discussed when we study Cauchy-Schwartz's inequality (when the equality holds).
						\item
						If there is a line, $y=ax+b$ with $a \neq 0$, such that the values of the 2-dimensional random vector $(X,Y)$ have a high probability of being near this line, then
						the correlation between $X$ and $Y$  will be near 1 or $-1$.
					\end{itemize}
				\end{block}
			}
			\frame{\vspace{-2mm}
				\begin{block}{}
					On the other hand, $X$ and $Y$ may be highly related but have no linear relationship, and the correlation could be nearly 0.
				\end{block}\vspace{-2mm}
				\begin{alertblock}{Example 4.5.8, 4.5.9.}
					Consider $(X,Y)$ having pdf \vspace{-2mm}
					\[ f(x,y) = \left\{ \br{ll}
					10 & \quad 0<x<1, \ x<y< x+0.1 \\
					0 & \quad \mbox{otherwise} \er \right. \vspace{-2mm} \]
					This is the same as the pdf in Example 4.5.4 except that $x<y<x+0.1$ (instead of $x<y<x+1$)
					in the region $\{ f(x,y)>0 \}$.
					
					The same calculation as in Example 4.5.4 shows that $\rho_{X,Y} = \sqrt{100/101}$, which is almost 1 (the first figure below).
					
					\vspace{-7.8cm}
					\begin{figure}[h]
						\begin{center}
							%\caption{Graph of $y=\sin^2(x)$ in Example 2.1.2.}
							\includegraphics[height=6in, width=4.5in]{figure405.pdf}
						\end{center}
					\end{figure}
					

				\end{alertblock} }
\frame{
\vspace{-2mm}
	\begin{alertblock}{} 
					Consider $(X,Y)$ having a different pdf
					(the 2nd figure) \vspace{-3mm}
					\[ f(x,y) = \left\{ \br{ll}
					5 & \quad -1<x<1, \ x^2 <y< x^2+0.1 \\
					0 & \quad \mbox{otherwise} \er \right. \vspace{-3mm} \]
					By symmetry, $E(X) = 0$ and $E(XY) = 0$; 
					hence, Cov$(X,Y) = \rho_{X,Y} = 0$.
					
					There is actually a strong relationship between $X$ and $Y$.
					
					But this relationship is not linear.
					
					The correlation does not measure any nonlinear relationship.
		
	\end{alertblock}
\vspace{-2.5mm}
	\begin{block}{}
	Whether random variables are related or not related at all is 
	described by the concept of independence. 
	\end{block}
	\vspace{-2.5mm}
\begin{alertblock}{First definition of independence of random variables/vectors}
Let $X_1,...,X_k$ be random variables, $F(x_1,...,x_k)$ be their joint cdf., and $F_{X_i}(x_i)$ be the marginal cdf of $X_i$, $i=1,...,k$.

$X_{1},...,X_k$ are (statistically) independent iff \vspace{-3mm}
\[ F(x_1,...,x_k) = F_{X_1}(x_1) \cdots F_{X_k}(x_k) \quad \quad (x_1,...,x_k) \in {\cal R}^k \vspace{-3mm}
\]
We define the independence of random vectors $X_1,...,X_k$ (they may have different dimensions) in the same way, except that
$F$ is the joint cdf of $(X_1,...,X_k)$ and $F_{X_i}$ is the (joint) cdf of $X_i$, $i=1,...,k$.
\end{alertblock}}
\frame{\vspace{-2mm}
\begin{alertblock}{}
Clearly, this definition is equivalent to the following definition.
\end{alertblock}\vspace{-2mm}
\begin{block}{Second definition of independence of random variables/vectors}
Random vectors $X_1,...,X_k$ are independent iff for any permutation $i_1,...,i_k$ of $1,...,k$, and any $r=1,...,k-1$,
the conditional distribution of $X_{i_1},...,X_{i_r}$ given $X_{i_{r+1}},...,X_{i_k}$ is the same as
the distribution of $X_{i_1},...,X_{i_r}$.
\end{block}\vspace{-2mm}
\begin{alertblock}{}
\begin{itemize}
\item
The joint cdf is determined by the $n$ marginal cdf's if $X_1,...,X_n$ are independent; otherwise, the joint cdf depends on marginal cdf's and conditional distributions.
 \vspace{-0.5mm}
\item
If $X_1,...,X_k$ are independent, then $X_{i_1},...,X_{i_r}$ are independent for any subest $\{ i_1,...,i_r \} \subset \{1,...,k\}$.
 \vspace{-0.5mm}
\item
When $k=2$ and both $X_1$ and $X_2$ are discrete random variables, they are independent iff all $p_{ij} = p_{i \cdot } p_{\cdot j}$ in the probability table.
 \vspace{-0.5mm}
\item
If $X_i$ has pmf (or pdf) $f_{X_i}$, $i=1,...,k$, then $X_1,...,X_k$ are independent iff  the joint pmf (or pdf) satisfies \vspace{-3mm}
\[ f(x_1,...,x_k) = f_{X_1}(x_1) \cdots f_{X_k}(x_k) \quad \quad (x_1,...,x_k) \in {\cal R}^k \vspace{-2mm} \]
In fact, sometimes the following lemma can be applied.
\end{itemize}
\end{alertblock}}
\frame{\vspace{-2mm}
\begin{block}{Lemma 4.2.7.}
Let $X$ and $Y$ be random variables having joint pmf or pdf $f(x,y)$.

$X$ and $Y$ are independent iff there exist functions $g(x)$ and $h(y)$ with \vspace{-2.5mm}
\[ f(x,y) = g(x)h(y) \quad \quad x\in {\cal R}, \ y\in {\cal R}  \vspace{-2.5mm}\]
A similar result holds for any fixed number of random variables/vectors.
\end{block}
\vspace{-2.5mm}
\begin{alertblock}{}
In the definition of independence of random vectors, the components of each random vector may be dependent or independent.

For example, if $X$, $Y$, and $Z$ are 3 random variables, we may have that $(X,Y)$ and $Z$ are independent but $X$ and $Y$ are not independent.
\end{alertblock}\vspace{-2.5mm}
\begin{alertblock}{Example}
 Suppose that $(X,Y,Z)$ has pdf \vspace{-3mm}
\[ f(x,y,z) = \left\{ \br{ll}
e^{-y} e^{-z}   & \quad 0<x<y , z > 0 \vspace{1mm} \\
0 & \quad \mbox{otherwise} \er \right. \vspace{-3mm} \]
It can be shown that $ f(x,y,z) = g(x,y)h(z)$ with\vspace{-3mm}
\[ g(x,y)=\left\{ \br{ll}
e^{-y}   & \ \ 0<x<y  \vspace{1mm} \\
0 & \ \ \mbox{otherwise} \er \right.
\quad \quad h(z)=\left\{ \br{ll}
e^{-z}   & \ \ z>0  \vspace{1mm} \\
0 & \ \ z\leq 0 \er \right. \]
\end{alertblock}}
\frame{\vspace{-2mm} 
\begin{alertblock}{}
Hence, $Z$ and $U=(X,Y)$ are independent.

In Example 4.2.4, we showed that $Y|X=x \sim exponential (x,1)$,

The marginal pdf of $Y$ is i.e., $ gamma(2,1)$, because \vspace{-2mm}
\[ \int_{-\infty}^\infty \int_{-\infty}^\infty f(x,y,z) dx dz =  \int_0^y e^{-y} dx = ye^{-y} \quad \quad y> 0 
\vspace{-2mm} \]
By the 2nd definition of independence, $X$ and $Y$ are not independent.

In fact, when $Y|X=x$ has a pdf varying with $x$,
we have already known that $X$ and $Y$ are not independent.

Consequently, $X$, $Y$, and $Z$ are not independent.
\end{alertblock}\vspace{-2mm}
\begin{block}{}
Note that ``$X$, $Y$, and $Z$ are independent" may be different from ``$(X,Y)$ and $Z$ are independent".

Hence, when random vectors are involved, it is important to specify which variables/vectors are independent.
\end{block}\vspace{-2mm}
\begin{alertblock}{Third definition of independence of random variables/vectors}
Random vectors $X_{1},...,X_k$ are independent iff events $\{ X_1 \in B_1 \}$, ...,$\{ X_k \in B_k \}$ are independent for any Borel sets $B_1,...,B_k$,
where the dimension of $B_i$ is the same as the dimension of $X_i$.
\end{alertblock}}
\frame{\vspace{-2mm}
\begin{block}{}
It can be shown that this definition is equivalent to the first  definition.

In fact, it is easy to show this definition implies the first definition, by considering $B_i = (-\infty , x_i]$.

 A rigorous proof is out of the scope of this course.
\end{block}
\vspace{-2mm} 
\begin{alertblock}{Theorem 4.6.12.}
If  $X_1,...,X_k$ are independent random vectors and $g_1,...,g_k$ are functions, then \\
(i) $g_1(X_1), ..., g_k(X_k)$ are independent, and \\
(ii) $E[g_1(X_1)\cdots g_k(X_k) ] = E[g_1(X_1)]\cdots E[g_k(X_k)] $.
\end{alertblock}\vspace{-2mm}
\begin{alertblock}{}
The following is an important corollary.
\end{alertblock}\vspace{-2mm}
\begin{block}{Theorem 4.5.5.}
If $X_1,...,X_k$ are independent random variables, then $X_i$ and $X_j$ are uncorrelated for every pair $(i,j)$.
\end{block}\vspace{-2mm}
\begin{alertblock}{}
The converse is not true, i.e., there are uncorrelated $X$ and $Y$ but they are not independent.

An example is $X$ and $Y$ in Example 4.5.9 with quadratic 
relationship. 
\end{alertblock}
}
\frame{\vspace{-2mm}
	
\begin{alertblock}{Example}
	Let $(X,Y)$ be a random vector on ${\cal R}^2$ with pdf \vspace{-3mm}
	\[ f(x,y) = \left\{ \br{ll}
	\pi^{-1} & \quad x^2+y^2 \leq 1 \vspace{1mm} \\
	0 & \quad x^2+y^2>1 . \er \right. \vspace{-2.5mm} \]
	Let $D = \{ (x,y): x^2+y^2 \leq 1 \}$.

By changing variables, we obtain that
\vspace{-3mm}
\beqrs
E(XY)& =& \pi^{-1} \int_{D} dxdy \\
& = & \pi^{-1} \left( \int_{D, xy>0 } dxdy
+  \int_{D, xy<0 } dxdy \right) \\
& = &  \pi^{-1} \left( \int_{D, xy>0 } dxdy
-  \int_{D, xy>0 } dxdy \right) \\
&= & 0
\eeqrs

\vspace{-3mm}
Similar, $EX=EY=0$ and, hence,
${\rm Cov}(X,Y)=0$.

\vspace{1mm} 

How do we show $X$ and $Y$ are not independent?
\vspace{1mm} 

It is unnecessary to derive the marginal distributions for showing that $X$ and $Y$ are not independent.
\end{alertblock}}
\frame{\vspace{-2mm}
\begin{alertblock}{}
By the 3rd definition of independence, we just need to find two Borel sets $A$ and $B$ such that \vspace{-2mm}
\[ P(X \in A, Y \in B) \neq P(X \in A)P(Y \in B) \vspace{-2mm} \]

A direct calculation shows that\vspace{-2mm}
\[ P( 0<X<1/\sqrt{2} , 0<Y<1/\sqrt{2} ) = \frac{1}{2\pi}\vspace{-2mm} \]
and\vspace{-2mm}
\[ P( 0<X<1/\sqrt{2} )
=P( 0<Y<1/\sqrt{2} ) = \frac{1}{4} + \frac{1}{2\pi} .\vspace{-2mm}\]
Thus, $X$ and $Y$ are not
independent because \vspace{-3mm}
\[ \frac{1}{2\pi} \neq \left(\frac{1}{4} + \frac{1}{2\pi}\right)^2 .\]
\end{alertblock}\vspace{-2mm}
\begin{block}{}
	\begin{itemize}
		\item
		This example indicates that if the distribution of $(X,Y)$ is symmetric in some way, then Cov$(X,Y)=0$ and
		$X$ and $Y$ don't have any linear relationship.
	\vspace{-0.5mm}
		\item
		Uncorrelated $X$ and $Y$ means that there is no {\bf linear} relationship between $X$ and $Y$, but $X$ and $Y$ may have a strong nonlinear relationship.
		\end{itemize}
\end{block}
}
\frame{
	\vspace{-2mm}
	\begin{block}{Example 4.5.10 (bivariate normal distribution).}
		Consider $(X,Y) \sim $ the bivariate normal distribution pdf on ${\cal R}^2$: \vspace{-2mm}
		\[ f(x,y)=  \frac{\exp \left( - \frac{(x - \mu_1)^2}{2 \sigma_1^2(1-\rho^2)}
			+ \frac{\rho (x-\mu_1)(y-\mu_2)}{\sigma_1\sigma_2(1-\rho^2)}
			- \frac{(y - \mu_2 )^2}{2 \sigma_2^2(1-\rho^2)} \right)}{2\pi \sigma_1\sigma_2 \sqrt{1-\rho^2}}\quad \quad (x,y) \in {\cal R}^2 \vspace{-2mm}\]
		where $\mu_1 \in {\cal R}$, $\mu_2\in {\cal R}$, $\sigma_1>0$, $\sigma_2>0$, and
		$-1 < \rho < 1$ are constants.
		\vspace{1mm}
		
		This 2-dimensional pdf has a very nice shape, as the following figure shows.
\vspace{-8.2cm}
			\begin{figure}[h]
				\begin{center}
					%\caption{Graph of $y=\sin^2(x)$ in Example 2.1.2.}
					\includegraphics[height=6in, width=5in]{figure400.pdf}
				\end{center}
			\end{figure}
		\end{block}}
		\frame{ \vspace{-2mm}
			\begin{block}{}
				The marginal pdf of $X$ is 
				 \vspace{-2mm}
				\beqrs
f_X(x) \back	& = & \back \frac{\exp \left( - \frac{(x - \mu_1)^2}{2 \sigma_1^2(1-\rho^2)}\right)}{2\pi \sigma_1\sigma_2 \sqrt{1-\rho^2}}
				\int_{-\infty}^\infty
				\exp \! \left( \! \frac{\rho (x-\mu_1)(y-\mu_2)}{\sigma_1\sigma_2(1-\rho^2)}
				- \frac{(y - \mu_2 )^2}{2 \sigma_2^2(1-\rho^2)} \! \! \right)\!\!  dy \\
			\back	& = & \back \frac{\exp \left( - \frac{(x - \mu_1)^2}{2 \sigma_1^2}\right)}{2\pi \sigma_1\sigma_2 \sqrt{1-\rho^2}}
				\int_{-\infty}^\infty
				\exp \left( - \frac{[(y - \mu_2 )-\frac{\rho\sigma_2}{\sigma_1} (x-\mu_1)]^2}{2 \sigma_2^2(1-\rho^2)} \right) dy \\
				\back & = &\back  \frac{1}{\sqrt{2\pi} \sigma_1} \exp \left( - \frac{(x - \mu_1)^2}{2 \sigma_1^2}\right)
				\eeqrs
				which is the pdf of $N(\mu_1,\sigma_1^2)$ (as expected).
				\vspace{1mm}
				
				Similarly, the marginal pdf of $Y$ is that of $N(\mu_2,\sigma_2^2)$.
				\vspace{1mm}
				
				We now calculate the correlation coefficient
				\vspace{-2mm}
				\beqrs
				\rho_{X,Y} \back & = & \back
				\frac{{\rm Cov}(X,Y)}{\sqrt{{\rm Var}(X){\rm Var}(Y)}} =
				E\left( \frac{X-\mu_1}{\sigma_1}\right)\left( \frac{Y-\mu_2}{\sigma_2} \right) \\
				\back & = & \back  \int_{-\infty}^\infty \int_{-\infty}^\infty \left( \frac{x-\mu_1}{\sigma_1}\right)\left( \frac{y-\mu_2}{\sigma_2} \right)
				f(x,y) dx dy
				\eeqrs
			\end{block}
		}
		\frame{\vspace{-2mm} 
			\begin{block}{}
				Letting $s= \left( \frac{x-\mu_1}{\sigma_1}\right)\left( \frac{y-\mu_2}{\sigma_2} \right)$ and $t =  \frac{x-\mu_1}{\sigma_1}$, we obtain \vspace{-2mm}
				\beqrs
				\rho_{X,Y} \back & = & \back \int_{-\infty}^\infty \int_{-\infty}^\infty s f\left( \sigma_1t+\mu_1,\sigma_2 \frac{s}{t} + \mu_2\right) \frac{\sigma_1\sigma_2}{|t|} ds dt \\
				\back & = & \back
				\int_{-\infty}^\infty \int_{-\infty}^\infty \frac{s \exp \left( - \frac{t^2 -2\rho s + s^2/t^2}{2(1-\rho^2)} \right)}{2\pi \sigma_1\sigma_2 \sqrt{1-\rho^2}} \frac{\sigma_1\sigma_2}{|t|} ds dt \\
				\back & = & \back
				\int_{-\infty}^\infty \frac{\exp (-t^2/2)}{2\pi\sqrt{(1-\rho^2)t^2}} \left[ \int_{-\infty}^\infty s \exp \left( - \frac{(s-\rho t^2)^2}{2(1-\rho^2)t^2} \right) ds\right]  dt \\
				\back & = & \back \int_{-\infty}^\infty \frac{\exp (-t^2/2)}{\sqrt{2\pi}} \rho t^2 dt
				=  \frac{\rho}{\sqrt{2\pi}} \int_{-\infty}^\infty t^2 \exp (-t^2/2) dt \\
				\back & = & \back \rho
				\eeqrs
				$~~$ \vspace{-6mm}
				
				Thus, all 5 parameters in $f(x,y)$ have meanings:
				
				$\mu_1$ and $\mu_2$ are two marginal means.
				
				$\sigma_1^2$ and $\sigma_2^2$ are two marginal variances.
				
				$\rho$ is the correlation coefficient for the two marginal variables.
				\vspace{1mm} 
				
				The conditional pdf of $Y|X$ can be derived as follows. 
			\end{block}}
\frame{	\vspace{-2mm} 
		
\begin{block}{}
\vspace{-4mm}
\[  \frac{f(x,y)}{f_X(x)}=  \frac{\exp \left( - \frac{(x - \mu_1)^2}{2 \sigma_1^2(1-\rho^2)}
	+ \frac{\rho (x-\mu_1)(y-\mu_2)}{\sigma_1\sigma_2(1-\rho^2)}
	- \frac{(y - \mu_2 )^2}{2 \sigma_2^2(1-\rho^2)} \right)}{2\pi \sigma_1\sigma_2 \sqrt{1-\rho^2}} \bigg/
\frac{\exp \left( - \frac{(x - \mu_1)^2}{2 \sigma_1^2}\right)}{\sqrt{2\pi} \sigma_1} \vspace{-2mm} \]
which is equal to \vspace{-2mm}
\[ \frac{\exp \left( - \frac{\rho^2 (x - \mu_1)^2}{2 \sigma_1^2(1-\rho^2)}
	+ \frac{\rho (x-\mu_1)(y-\mu_2)}{\sigma_1\sigma_2(1-\rho^2)}
	- \frac{(y - \mu_2 )^2}{2 \sigma_2^2(1-\rho^2)} \right)}{\sqrt{2\pi} \sigma_2 \sqrt{1-\rho^2}} =  \frac{\exp \left(
	- \frac{[y - \mu_2 - \frac{\rho \sigma_2}{\sigma_1} (x-\mu_1)]^2}{2 \sigma_2^2(1-\rho^2)} \right)}{\sqrt{2\pi} \sigma_2 \sqrt{1-\rho^2}} \vspace{-2mm} \]
Hence, \vspace{-2mm}
\[ Y|X \sim N\left(\mu_2 + \frac{\rho \sigma_2}{\sigma_1} (X-\mu_1), \sigma_2^2(1-\rho^2)\right) \vspace{-2mm}
\]
From the second definition of independence,  $X$ and $Y$ 
are independent if they are uncorrelated ($\rho =0$).
\vspace{1mm}

On the other hand, if $\rho \neq 0$, then $X$ and $Y$ are correlated and they cannot be independent (Theorem 4.5.5).
\vspace{1mm}

This means that for bivariate normal $(X,Y)$, 
independence of $X$ and $Y$ is equivalent to $\rho =0$:
either $X$ and $Y$ have a linear relationship or they don't have any relationship at all
(they cannot have a nonlinear relationship).

\end{block}
}
\end{document}

\frame{
\begin{alertblock}{}
Sometimes there is an extremely strong relationship between $X$ and $Y$, but the correlation is still 0 when
the relationship is not linear.
\end{alertblock}
\vspace{-2mm}
\begin{alertblock}{Example}
Let $X$ be a random variable with $E(X^2) < \infty$ and let $Y=|X|$.

Suppose that $X$ has a pdf $f_X$
symmetric about 0, i.e., $f_X(x)=f_X(-x)$.

Since $X$ and $XY=X|X|$ are odd functions of $X$,
$EX=0$ and $E(XY)=E(X|X|)=0 $ and, thus,
Cov$(X,Y) = 0$.

Let $t$ be a positive constant such that $p= P(0 < X < t) >0$.\vspace{-2mm}
\beqrs
  P(0 < X < t, Y<t) &=&  P(0<X<t,-t<X<t) \\
& = & P(0<X<t)  = p \\
 P(0<X<t)P(Y<t)& =&
 P(0<X<t)P(-t<X<t) \\
&= &2P(0<X<t)P(0<X<t) \\
&=& 2p^2
\eeqrs
i.e., $P(0<X<t,Y<t) \neq P(0<X<t)P(Y<t)$.

Hence $X$ and $Y$ are not
independent.
\end{alertblock}
}

