% This is a simple LaTex sample document that gives a submission format
%   for IEEE PAMI-TC conference submissions.  Use at your own risk.

% Make two column format for LaTex 2e.
\documentclass[11pt,twocolumn]{article} %,twocolumn
\usepackage[dvips]{graphicx,graphics}
\usepackage{times,amsmath,amsfonts}

% Use following instead for LaTex 2.09 (may need some other mods as well).
% \documentstyle[times,twocolumn]{article}

% Set dimensions of columns, gap between columns, and paragraph indent
\setlength{\textheight}{9.9in} \setlength{\textwidth}{6.9in}
%\setlength{\columnsep}{0.3125in} \setlength{\topmargin}{0in}
\setlength{\headheight}{0in} \setlength{\headsep}{-1in}
\setlength{\parindent}{1pc}
\setlength{\oddsidemargin}{-.2in}  % Centers text.
\setlength{\evensidemargin}{-.2in}

% Add the period after section numbers.  Adjust spacing.
\newcommand{\Section}[1]{\vspace{-8pt}\section{\hskip -1em.~~#1}\vspace{-3pt}}
\newcommand{\SubSection}[1]{\vspace{-3pt}\subsection{\hskip -1em.~~#1}
        \vspace{-3pt}}
\newcommand{\bqn}{\begin{eqnarray}}
\newcommand{\eqn}{\end{eqnarray}}
\newcommand{\diff}[1] {\frac{\partial}{\partial #1}}
\newcommand{\jacob}[3]{\frac{\partial^2 #3}{\partial #1 \partial #2}}
\newcommand{\der}[2]{\frac{\partial #2}{\partial #1}}
\begin{document}

% Don't want date printed
%\date{January 23, 2003}

% Make title bold and 14 pt font (Latex default is non-bold, 16pt)
\title{Stat 312: Lecture 21\\
Linear Regression II.}

% For single author (just remove % characters)
\author{Moo K. Chung\\
mchung@stat.wisc.edu}
\date{Dec 7, 2004}
% For two authors (default example)
\maketitle \thispagestyle{empty} \subsection*{Concepts}
\begin{enumerate}

\item {\em Maximum likelihood estimation.} Given a linear model
$$Y_j = \beta_0 + \beta_1 x_j + \epsilon_j$$
with $\epsilon_j \sim N(0,\sigma^2)$, we will estimate
$\beta_0,\beta_1$ using the maximum likelihood estimation. Note that
$$Y_j \sim N(\beta_0 + \beta_1 x_j, \sigma^2).$$ The density function for $Y_j$ are
$$f(y_j) = \frac{1}{\sqrt{2\pi}\sigma} \exp \Big[\frac{(y_j -\beta_0
-\beta_j x_j)^2}{2\sigma^2}\Big].$$ The loglikelihood is
$$L(\beta_0,\beta_1) = const - \frac{1}{2\sigma^2}\sum_{j=1}^n (y_j
- \beta_0 - \beta_1 x_j)^2.$$ We maximize the loglikelihood which is
equivalent to minimizing the sum of the residuals
$$\frac{1}{2\sigma^2}\sum_{j=1}^n (y_j
- \beta_0 - \beta_1 x_j)^2.$$ Hence MLE is LSE in linear regression.

\item The least squares estimation for $\beta_1$ are given by
$$\hat \beta_1 =
\frac{S_{xy}}{S_{xx}}$$ where $S_{xy} = n(\overline{xy}
-\bar{x}\bar{y})$. It can be shown that $$\hat \beta_1 =
\frac{S_{xy}}{S_{xx}}=\sum_{j=1}^n c_jY_j,$$ where
$$c_j = (x_j -\bar x)/S_{xx}.$$ So LSE is a linear estimation. From $\sum_{j=1}^n c_j = 0$, $\sum_{j=1}^n c_jx_j=1$ and
$\sum_{j=1}^n c_j^2 = S_{xx}^{-1}$ we can show that
$$\mathbb{E} \hat \beta_1 = \beta_1$$
showing unbiasness. Further, $\mathbb{V} \hat \beta_1 =
\sigma^2/S_{xx}$. Since $\hat \beta_1$ is a linear combination of
normals, $\hat \beta_1 \sim N(\beta_1, \sigma^2/S_{xx})$.


\item We are interested in hypothesis testing
$$H_0: \beta_1 = 0 \mbox{ vs. } H_1:\beta_1 \neq 0.$$
Inference on the slope parameter $\beta_1$ is based on test
statistic
$$T =
\frac{\hat \beta_1 - \beta_1}{S_{\hat \beta_1}} \sim t_{n-2},$$
where $S_{\hat \beta_1} = \hat \sigma/\sqrt{S_{xx}}$ and $\hat
\sigma = SSE/(n-2)$. It can be shown that $SSE =
S_{yy}-S_{xy}^2/S_{xx}$, we get $S_{\hat \beta_1} =
\frac{1}{\sqrt{n-2}}\sqrt{\frac{S_{yy}}{S_{xx}}-(\frac{S_{xy}}{S_{xx}})^2}$.
Then we reject $H_0$ if $|t|>t_{\alpha/2,n-2}$ at $100(1-\alpha)\%$
significance. We don't usually compute the test statistic by hand.
Use {\tt R}-package.
\end{enumerate}

{\em Example.} We continue Lecture 19 example.
\begin{verbatim}
>summary(lm(y~x))

Call: lm(formula = y ~ x)

Residuals:
    Min      1Q  Median      3Q     Max
-10.908  -6.312   1.758   4.354  10.836

Coefficients:
   Estimate Std. Error t value Pr(>|t|)

(Intercept) 29.48 13.23 2.22 0.06
x            0.55  0.17 3.12 0.01 *
---
Signif. codes:  0 `***' 0.001 `**' 0.01 `*' 0.05 `.' 0.1 ` ' 1

Residual standard error: 7.647 on 8 degrees
of freedom Multiple R-Squared: 0.5519,
Adjusted R-squared:0.4959 F-statistic:9.854on
1 and 8 DF, p-value: 0.01383

\end{verbatim}
\end{document}