\documentclass[a4wide]{article}
\usepackage{latexsym,amsfonts, a4wide,amsmath}
\usepackage{graphicx}

\title{Calculus: Homework 4} 
\author{}
\date{}
\begin{document}

\maketitle

\begin{enumerate}
\item (1.5 pt) Calculate the second-degree Taylor polynomial of
\[
f(x,y)=e^{(x^2+y^2)}
\]
at the point $(0,0)$.


\item (1.5 pt) Find the gradient of the following functions at the given point $p$
\begin{itemize}
    \item $f(x,y) = (ax^2 - x^2 - y^2)^{1/2}$ at $p=(a/2, a/2)$
    \item $g(x,y) = e^y \cos(3x+y)$ at $p=(\frac{2\pi}{3},0)$
    \item $h(x_1,\dots,x_n) = \sum_{i=1}^n x_1 x_i$ at $p=(1,\dots, 1)$
\end{itemize}

\item (1.5 pt) Find $\frac{\partial f}{\partial x}$ and  $\frac{\partial f}{\partial y}$ for the following functions
\begin{itemize}
    \item $x \cos(x) \sin(y)$
    \item $(x^2 + y^2) \ln(x^2+y^2)$
\end{itemize}

\item (1.5 pt) For all $1 \leq j \leq n$ find the partial derivative $\frac{\partial f}{\partial x_j}$ of the following functions
\begin{itemize}
    \item $f(x_1, \dots,x_n)= \Pi_{i=1}^n x_i^i$
    \item $g(x_1, \dots,x_n)= \sum_{i=2}^n (x_{i-1} +x_i)$
\end{itemize}


\item (2 pt) A {\em multivariate Gaussian} is a probability distribution of the form
\[
p(x_1, \dots, x_n) = \frac{1}{(2 \pi)^{n/2}  \mathrm{det}(\Sigma)^{1/2}} e^{-\frac{1}{2}(x - u)^T \Sigma^{-1} (x - u)}
\]
where $\Sigma$ is a symmetric $n\times n$ matrix, $u=(u_1, \dots,u_n)$ is a vector of size $n$ and $\vec{x} = (x_1,\dots,x_n)$. As usual, we denote by $A^T$ the transpose of a matrix $A$ and $\mathrm{det}(\Sigma)$ is the determinant of $\Sigma$. Note that
\[
-\frac{1}{2}(x - u)^T \Sigma^{-1} (x - u)
\] is a scalar.
Compute 
\[ 
\frac{\partial p}{\partial x_1} 
\]  in the case where $\Sigma$ is a diagonal matrix $\textrm{diag} (\sigma_1^2,\ldots,\sigma_n^2)$:
$$
\begin{bmatrix}
\sigma_1^2 &0 & \cdots & 0 \\
0 & \sigma_2^2 &  \cdots & 0 \\
\vdots & \vdots &  \ddots & \vdots \\
0 & 0 &  \cdots & \sigma_n^2 \\
\end{bmatrix}
$$



Hint: it is helpful to first rewrite $p$ such that it does not contain any matrix operation anymore. First prove that
\[
-\frac{1}{2}(x - u)^T \Sigma^{-1} (x - u) = -\frac{1}{2} \sum_{i=1}^n \frac{1}{\sigma_i^2}(x_i - u_i)^2
\]
The expression $(x - u)^T \Sigma^{-1} (x - u)$ is a nice and compact way to write the right-side sum, but it makes it difficult to compute the partial derivative. This is the reason why we first develop the matrix product. After doing this, calculate the partial derivative in the usual way.

\item (1 pt) We consider a probability distribution $p(x_1,\dots,x_n)$ defined over the set 

$\{ (x_1, \dots, x_n) : x_1 >0, \dots, x_n>0 \}$ by
\[
p(x_1, \dots,x_n) = \Pi_{i=1}^n p_i(x_i)
\]
with
\[
p_i(x_i) = \frac{\beta^\alpha}{\Gamma(\alpha)} x_i^{\alpha -1} e^{-\beta x_i}
\]
where $\alpha$ and $\beta$ are constants and $\Gamma$ is a function from $\mathbb{R}$ to $\mathbb{R}$, which you can regard as a constant as well when calculating the partial derivative (for your information, $p_i(x_i)$ is a so-called Gamma distribution).

For all $1 \leq j \leq n$, find 
\[
\frac{\partial p}{\partial x_j}
\]

\item (1 pt) In neural networks, the \textit{softmax function} is often used as the last layer. This function gets as input a vector of real numbers and rescales them so that the resulting vector sums to 1 (and can be interpreted as a probability distribution).

More formally, suppose you are given $n$ numbers: $\vec{z} = (z_1, z_2, \dots, z_n) \in \mathbb{R}^n$. The softmax function then calculates for each $j \in \{1, \dots, n\}$:
$$ p_j = \frac{e^{z_j}}{\sum_{k=1}^n e^{z_k}}$$

Calculate the partial derivatives of this softmax function with respect to the data points $z_i$. That is, compute for $1 \le i \le n$:
$$\frac{\partial p_j}{\partial{z_i}} $$

Hint: consider the case where $i = j$ separately from the case where $i \neq j$.

\iffalse \item Suppose that $f$ is a  function $f$ from $\mathbb{R}^k$ to $\mathbb{R}^n$.  So
\[
f(x_1,\dots, x_n) = (f_1(x_1,\dots, x_n), \dots, f_k(x_1,\dots, x_n))
\]
The jacobian matrix $Df(x,y)$  is defined by

$$
\begin{bmatrix}
\frac{\partial f_1}{\partial x_1} & \dots &  \frac{\partial f_1}{\partial x_k}\\
\frac{\partial f_2}{\partial x_1} & \dots &  \frac{\partial f_2}{\partial x_k}\\
\dots & \dots & \dots \\
\frac{\partial f_n}{\partial x_1} & \dots &  \frac{\partial f_n}{\partial x_k}\\
\end{bmatrix}
$$

Find the jacobian matrix for the following function from $\mathbb{R}^3$ to $\mathbb{R}^2$
\[
f(x,y,z) = (\sin(x)yz, xz)
\]
\fi
\end{enumerate}



\end{document}
