\documentclass[12pt]{article} 
%\usepackage{amsbsy} % for \boldsymbol and \pmb 
\usepackage{graphicx} % To include pdf files!
\usepackage{amsmath}
\usepackage{amsbsy}
\usepackage{amsfonts}
\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=blue]{hyperref} % For links
\usepackage{fullpage}
%\pagestyle{empty} % No page numbers


\begin{document}
%\enlargethispage*{1000 pt} 

\begin{center}   
{\Large \textbf{STA 312s19 Assignment One (Review)}}\footnote{This assignment was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Mathematical and Computational Sciences, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/312s19} {\texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/312s19}}}
\vspace{1 mm}
\end{center}

\noindent
The questions on this assignment are not to be handed in. They are practice for Quiz 1 on January 14th. Please see your textbook from STA256 and STA260 as necessary.

\begin{enumerate} 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% STA256 Review. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 

\item Recall the definition of a derivative: $\frac{d}{dx}f(x) = \lim_{\Delta \rightarrow 0}\frac{f(x+\Delta)-f(x)}{\Delta}$. 
    \begin{enumerate}
        \item Prove $\frac{d}{dx} x^2 = 2x$. 
        \item Let $a$ be a constant. Prove $\frac{d}{dx} \, a \, f(x) = a \,\frac{d}{dx}f(x)$. To do this with confidence, let $g(x) = a \, f(x)$, and note $g(x+\Delta) = a \, f(x+\Delta)$.
    \end{enumerate} 

% See sta256f18 Test 2 Question 5
\item The random variable $X$ has probability density function $f_x(x) = \frac{e^x}{(1+e^x)^2}$, for all real $x$. 
    \begin{enumerate}
        \item What is the cumulative distribution function $F_x(x) = P(X \leq x)$? Show your work. Answer: $1-\frac{1}{1+e^x}$.
        \item The median of a distribution is that point $m$ for which $P(X \leq m) = \frac{1}{2}$. What is the median of the distribution in this question? Answer: $m = 0$.
    \end{enumerate} 

% STA256 Cont RV sample Q2
\item Let $F(x) = P(X \leq x) = \left\{ \begin{array}{ll} % ll means left left
     0 & \mbox{for $ x < 0$}  \\
   x^\theta & \mbox{for $ 0 \leq x \leq 1$}  \\
   1     & \mbox{for } x>1 
   \end{array}  \right. $  % Need that crazy invisible right period!

    \begin{enumerate}
        \item If $\theta=3$, what is $P\left(\frac{1}{2} < X \leq 4\right)$? The answer is a number.  (Answer: $\frac{7}{8}$.)
        \item Find $f(x)$. Your answer must apply to all real $x$.
    \end{enumerate} 

\item The discrete random variables $X$ and $Y$ have joint distribution 
\begin{center}
\begin{tabular}{c|ccc} 
        &  $x=1$   & $x=2$    & $x=3$   \\  \hline
$y=1$   &  $3/12$  & $1/12$   & $3/12$  \\
$y=2$   &  $1/12$  & $3/12$   & $1/12$  \\ 
\end{tabular}
\end{center} 
    \begin{enumerate}
        \item What is $p_x(x)$, the marginal probability mass function of $X$?
        \item What is the conditional probability mass function of $X$ given $Y=1$?
        \item What is $E(X|Y=1)$? (Answer: 2)
    \end{enumerate}

\pagebreak

% STA256 Cont RV sample Q5 except for MGF.
\item \label{expo} The Exponential($\lambda$) distribution has density
    $f(x) = \left\{ \begin{array}{ll} % ll means left left
   \lambda e^{-\lambda x}   & \mbox{for $x \geq 0$}  \\
   0     & \mbox{for } x < 0 
   \end{array}  \right.$, \\ % Need that crazy invisible right period!
where $\lambda>0$.
    \begin{enumerate}
        \item Show $\int_{-\infty}^\infty f(x) \, dx = 1$.  
        \item Find $F(x)$. Of course there is a separate answer for $x \geq0$ and $x < 0$.
        \item Let $X$ have an exponential density with parameter $\lambda>0$. Prove the ``memoryless" property:
\begin{displaymath}
    P(X>t+s|X>s) = P(X>t)
\end{displaymath}
for $t>0$ and $s>0$. For example, the probability that the conversation lasts at least $t$ \underline{more} minutes is the same as the probability of it lasting at least $t$ minutes in the first place.
        \item Calculate the moment-generating function of an exponential random variable and use it to obtain the expected value.
    \end{enumerate} 


% This one is "new."
\item The continuous random variables $X$ and $Y$ have joint density
\begin{displaymath}
f_{x,y}(x,y) = \left\{ \begin{array}{ll} 
   2e^{-(x+2y)} & \mbox{for $ x \geq 0$ and $y \geq 0$}  \\
   0     & \mbox{otherwise}  
   \end{array}  \right.
\end{displaymath}
Find $P(X>Y)$. (Answer: $\frac{2}{3}$)


% STA256f18 Test 2 Q6
\item The continuous random variables $X$ and $Y$ have joint probability density function
\begin{displaymath}
    f_{xy}(x,y) = \left\{ \begin{array}{ll} % ll means left left
   10 \, x^2y & \mbox{for $0 \leq x \leq 1$ and $0 \leq y \leq x$}  \\
   0     & \mbox{otherwise}  
   \end{array}  \right. % Need that crazy invisible right period!
\end{displaymath}
Find the marginal density function $f_y(y)$. Show your work. Do not forget to indicate where the density is non-zero.

% STA256f18 Cont. RVs sample Q6
\item The Gamma($\alpha,\lambda$) distribution has density 
    $f(x) = \left\{ \begin{array}{ll} % ll means left left
   \frac{\lambda^\alpha}{\Gamma(\alpha)} e^{-\lambda x} \, x^{\alpha-1}  & \mbox{for $x \geq 0$}  \\
   0     & \mbox{for } x < 0 
   \end{array}  \right.$,  % Need that crazy invisible right period!
where $\alpha>0$ and $\lambda>0$.
    \begin{enumerate}
        \item Show $\int_{-\infty}^\infty f(x) \, dx = 1$. 
Recall $\Gamma(\alpha) = \int_0^\infty e^{-t} t^{\alpha-1} \, dt$. 
        \item If $X$ has a gamma distribution with parameters $\alpha$ and $\lambda$, find a general expression for $E(X^k)$. (Answer: $\frac{\Gamma(\alpha+k)}{\Gamma(\alpha)\lambda^k}$.)
        \item Use your answer to the last question to find $Var(X)$. The identity  $\Gamma(\alpha+1)=\alpha\Gamma(\alpha)$ will help. 
    \end{enumerate}

\item Let $X$ have an exponential distribution with $\lambda=1$ (see Question~\ref{expo}), and let $Y=\log(X)$. Find the probability density function of $Y$. Where is the density non-zero? Note that in this course, $\log$ refers to the log base $e$, or natural log, often symbolized $\ln$. The distribution of $Y$ is called the (standard) Gumbel, or extreme value distribution.

\pagebreak

\item The Normal($\mu,\sigma^2$) distribution has density
$f(x) = \frac{1}{\sigma \sqrt{2\pi}}\exp\left\{{-\frac{(x-\mu)^2}{2\sigma^2}}\right\}$, where \\ $-\infty < \mu < \infty$ and $\sigma>0$. Let the random variable $T$ be such that $X = \log(T)$ is Normal($\mu,\sigma^2$). Find the density of $T$. This distribution is known as the \emph{log normal}.  Do not forget to indicate where the density of $T$ is non-zero.


%%%%%%%%% Max Likelihood by hand - Review (See openSEM text for exposition) %%%%%%%%%%%%%%

\item Choose the correct answer.
    \begin{enumerate}
    \item $\prod_{i=1}^n e^{x_i}=$
        \begin{enumerate}
            \item $\exp(\prod_{i=1}^n x_i)$
            \item $e^{nx_i}$
            \item $\exp(\sum_{i=1}^n x_i)$
        \end{enumerate}

    \item $\prod_{i=1}^n \lambda e^{-\lambda x_i}=$
        \begin{enumerate}
            \item $\lambda e^{-\lambda^n x_i}$
            \item $\lambda^n e^{-\lambda n x_i}$
            \item $\lambda^n \exp(-\lambda \sum_{i=1}^n x_i)$
            \item $\lambda^n \exp(-n\lambda \sum_{i=1}^n x_i)$
            \item $\lambda^n \exp(-\lambda^n \sum_{i=1}^n x_i)$
        \end{enumerate}

    \item $\prod_{i=1}^n a_i^b=$
        \begin{enumerate}
            \item $n a_i^b$
            \item $a_i^{nb}$
            \item $(\prod_{i=1}^n a_i)^b$
        \end{enumerate}

    \item $\prod_{i=1}^n a^{b_i}=$
        \begin{enumerate}
            \item $n a^{b_i}$
            \item $a^{n b_i}$
            \item $\sum_{i=1}^n a^{b_i}$
            \item {\Large$a^{\prod_{i=1}^n b_i}$} 
            \item {\Large$a^{\sum_{i=1}^n b_i}$}
        \end{enumerate}

    \item $\left( e^{\lambda(e^t-1)} \right)^n = $
        \begin{enumerate}
            \item $n e^{\lambda(e^t-1)}$
            \item $e^{n\lambda(e^t-1)}$
            \item $e^{\lambda(e^{nt}-1)}$
            \item $e^{n\lambda(e^{t}-n)}$
        \end{enumerate}

    \item $\left(\prod_{i=1}^n e^{-\lambda x_i}\right)^2=$
        \begin{enumerate}
            \item $e^{-2n\lambda x_i}$
            \item $e^{-2\lambda \sum_{i=1}^n x_i}$
            \item $2e^{-\lambda \sum_{i=1}^n x_i}$
        \end{enumerate}
    \end{enumerate}

\pagebreak

    \item True, or False?
        \begin{enumerate}
            \item $\sum_{i=1}^n \frac{1}{x_i} =  \frac{1}{\sum_{i=1}^n x_i}$
            \item $\prod_{i=1}^n \frac{1}{x_i} =  \frac{1}{\prod_{i=1}^n x_i}$
            \item $\frac{a}{b+c}=\frac{a}{b}+\frac{a}{c}$
            \item $\ln(a+b) = \ln(a) + \ln(b)$
            \item $e^{a+b} = e^a + e^b$
            \item $e^{a+b} = e^a  e^b$
            \item $e^{ab} = e^a  e^b$
            \item $\prod_{i=1}^n (x_i+y_i) = \prod_{i=1}^n x_i +  \prod_{i=1}^n y_i$
            \item $\ln (\prod_{i=1}^n a_i^b) = b \sum_{i=1}^n \ln(a_i)$
            \item $\sum_{i=1}^n \prod_{j=1}^n a_j = n \prod_{j=1}^n a_j$
            \item $\sum_{i=1}^n \prod_{j=1}^n a_i = \sum_{i=1}^n a_i^n$
            \item $\sum_{i=1}^n \prod_{j=1}^n a_{i,j} = 
                   \prod_{j=1}^n \sum_{i=1}^n a_{i,j}$
        \end{enumerate}

    \item Simplify as much as possible.
        \begin{enumerate}
            \item  $\ln \prod_{i=1}^n \theta^{x_i} (1-\theta)^{1-{x_i}}$
            \item  $\ln \prod_{i=1}^n \binom{m}{{x_i}} \theta^x (1-\theta)^{m-x_i}$
            \item  $\ln \prod_{i=1}^n \frac{e^{-\lambda}\lambda^{x_i}}{x_i!}$
            \item  $\ln \prod_{i=1}^n \theta (1-\theta)^{x_i-1}$
            \item  $\ln \prod_{i=1}^n \frac{1}{\theta} e^{-x_i/\theta}$
            \item  $\ln \prod_{i=1}^n \frac{1}{\beta^\alpha \Gamma(\alpha)} 
                    e^{-x_i/\beta} x_i^{\alpha - 1}$
            \item  $\ln \prod_{i=1}^n \frac{1}{2^{\nu/2}\Gamma(\nu/2)}
                    e^{-x_i/2} x_i^{\nu/2 - 1}$
            \item  $\ln \prod_{i=1}^n \frac{1}{\sigma \sqrt{2\pi}}
                    e^{-\frac{(x_i-\mu)^2}{2 \sigma^2}}$
%            \item  $\prod_{i=1}^n \frac{1}{\beta-\alpha} 
%                    I(\alpha \leq x_i \leq \beta)$ (Express in terms of the minimum and maximum $y_1$ and $y_n$.)
        \end{enumerate} 

\item For each of the following distributions, derive a general expression for the Maximum Likelihood Estimator (MLE). Carry out the second derivative test to make sure you really have a maximum. Then use the data to calculate a numerical estimate.
    \begin{enumerate}
        \item $p(x)=\theta(1-\theta)^x$ for $x=0,1,\ldots$, where $0<\theta<1$. Data: \texttt{4, 0, 1, 0, 1, 3, 2, 16, 3, 0, 4, 3, 6, 16, 0, 0, 1, 1, 6, 10}. Answer: 0.2061856   
                % Geometric .25, thetahat = 1/xbar 
        \item $f(x) = \frac{\alpha}{x^{\alpha+1}}$ for $x>1$, where $\alpha>0$. Data: \texttt{1.37, 2.89, 1.52, 1.77, 1.04, 2.71, 1.19, 1.13, 15.66, 1.43} Answer: 1.469102
                % Pareto alpha = 1 (one over uniform) alphahat = 1/mean(log(x))
        \item $f(x) = \frac{\tau}{\sqrt{2\pi}} e^{-\frac{\tau^2 x^2}{2}}$, for $x$ real, where $\tau>0$. Data: \texttt{1.45, 0.47, -3.33, 0.82, -1.59, -0.37, -1.56, -0.20 } Answer: 0.6451059 % Normal mean zero   tauhat = sqrt(1/mean(x^2))

        \item $f(x) = \frac{1}{\theta} e^{-x/\theta}$ for $x>0$, where $\theta>0$. Data: \texttt{0.28, 1.72, 0.08, 1.22, 1.86, 0.62, 2.44, 2.48, 2.96} Answer: 1.517778  % Exponential, true theta=2, thetahat = xbar
    \end{enumerate}
% \item Each of the above, start with second derivative test, get SE. 
% \item Normal random sample, numerical n = 50 or so, CI all with R. What is diff from usual CI for mu? For sigmasquared? This is a good problem. 

%%%%%%%%% End of Max Likelihood by hand - Review %%%%%%%%%%%%%%




\end{enumerate} % End of all questions


\end{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% STA256 Cont RV sample Q1 
\item The continuous random variable $X$ has density 
$    f(x) = \left\{ \begin{array}{ll} % ll means left left
   \frac{c}{x^{\alpha+1}} & \mbox{for $ x \geq 1$}  \\
   0     & \mbox{for } x<1 
   \end{array}  \right. $  % Need that crazy invisible right period!
\\ where $\alpha > 0$.

    \begin{enumerate}
        \item Find the constant $c$. (Answer: $c=\alpha$.)
        \item Find the cumulative distribution function $F(x)$. The answer must apply to all real $x$, so consider $x<1$ and $x\ge 1$ separately.
        \item The median of this distribution is that point $m$ for which $P(X \leq m) = \frac{1}{2}$. What is the median? The answer is a function of $\alpha$. (Answer: $m = 2^{1/\alpha}$.)
    \end{enumerate} 



% STA256 joint distributions sample Q
\item The continuous random variables $X$ and $Y$ have joint density
\begin{displaymath}
f_{x,y}(x,y) = \left\{ \begin{array}{ll} 
   2e^{-(x+y)} & \mbox{for $ 0 \leq x \leq y$ and $y \geq 0$}  \\
   0     & \mbox{otherwise}  
   \end{array}  \right.
\end{displaymath}
Obtain the marginal density $f_x(x)$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

I have noticed that a major obstacle for many students when doing maximum likelihood calculations is a set of basic mathematical operations they actually know. But the mechanics are rusty, or the notation used in Statistics is troublesome. So, with sincere apologies to those who don't need this, here are some basic rules. 
\begin{itemize}
     \item The distributive law: $a(b+c)=ab+ac$. You may see this in a form like
            \begin{displaymath}
            \theta \sum_{i=1}^n x_i = \sum_{i=1}^n \theta x_i
            \end{displaymath}
     \item Power of a product is the product of powers: $(ab)^c = a^c \, b^c$. You may see this in a form like
            \begin{displaymath}
            \left(\prod_{i=1}^n x_i\right)^\alpha =  \prod_{i=1}^n x_i^\alpha
            \end{displaymath}
     \item Multiplication is addition of exponents: $a^b a^c = a^{b+c}$. You may see this in a form like
            \begin{displaymath}
            \prod_{i=1}^n \theta e^{-\theta x_i} = 
            \theta^n \exp(-\theta \sum_{i=1}^n x_i)
            \end{displaymath}
     \item Powering is multiplication of exponents: $(a^b)^c = a^{bc}$. You may see this in a form like
            \begin{displaymath}
            (e^{\mu t + \frac{1}{2}\sigma^2 t^2})^n = 
            e^{n\mu t + \frac{1}{2}n\sigma^2 t^2}
            \end{displaymath}
     \item Log of a product is sum of logs: $\ln(ab) = \ln(a)+\ln(b)$. You may see this in a form like
            \begin{displaymath}
            \ln \prod_{i=1}^n x_i = \sum_{i=1}^n \ln x_i
            \end{displaymath}
     \item Log of a power is the exponent times the log: $\ln(a^b)=b\,\ln(a)$. You may see this in a form like
            \begin{displaymath}
            \ln(\theta^n) = n \ln \theta
            \end{displaymath}
     \item The log is the inverse of the exponential function: $\ln(e^a) = a$. You may see this in a form like
            \begin{displaymath}
            \ln\left(  \theta^n \exp(-\theta \sum_{i=1}^n x_i) \right) =
            n \ln \theta - \theta \sum_{i=1}^n x_i
            \end{displaymath}
\end{itemize}



