\documentclass[11pt]{article}
\usepackage{fullpage}
\usepackage{graphicx}
\usepackage{float}
\usepackage{amssymb,amsmath}
\usepackage[colorlinks,linkcolor=blue,urlcolor=blue,citecolor=black]{hyperref} % Create hyperlinks within the document
\usepackage{url}

\usepackage{listings}
\usepackage{color}

\definecolor{dkgreen}{rgb}{0,0.6,0}
\definecolor{gray}{rgb}{0.5,0.5,0.5}
\definecolor{mauve}{rgb}{0.58,0,0.82}

\usepackage{caption}
\DeclareCaptionType{code}[Code Listing][List of Code Listings] 

\lstset{frame=tb,
  language=python,
  aboveskip=3mm,
  belowskip=3mm,
  showstringspaces=false,
  columns=flexible,
  basicstyle={\small\ttfamily},
  numbers=none,
  numberstyle=\tiny\color{gray},
  keywordstyle=\color{blue},
  commentstyle=\color{dkgreen},
  stringstyle=\color{mauve},
  breaklines=true,
  breakatwhitespace=true
  tabsize=3
}



%homework
%\long\def\comment#1{}
%solution
\long\def\comment{}

\newcommand{\lit}[1]{\fbox{\texttt{#1}}}

\begin{document}

\begin{enumerate}
  \item Solve x by hand.
  $$
  \left(1 1\right) \left( \begin{matrix}
  1 & 2\\
  3 & x
  \end{matrix}\right)
  \left(\begin{matrix}
  1\\
  1
  \end{matrix}\right) = x^2
  $$
  
  \item Compute the derivative (with respect to x) of the function
  $$
  \frac{1}{1+e^{-x}}
  $$
  \item  Find the minimum of the function $f(x, y) = x+y$, where $(x, y)$ must
be on the unit circle.

\item Let x be a random variable drawn from a Gaussian distribution with
mean 0 and variance $1/2\lambda$. Write down the expression for $\log p(x)$.
\item Download the particular version of Alice¡¯s Adventures in Wonderland from
\url{alice.txt}. This
is the document we¡¯ll be working on.
\begin{enumerate}
  \item 
Sentence Segmentation. Download MXTERMINATOR, a sentence
boundary detector, from \url{
jmx.tar.gz}. Follow the instruction in MXTERMINATOR.
html. If you use tcsh, simply do
setenv CLASSPATH mxpost.jar
then you should be able to run it. Use the eos.project that comes
with the package. Apply it to Alice.
 
\item
Tokenization. Once you have segmented out sentences, it¡¯s time
to separate individual words. Download the Penn Treebank tokenizer
from \url{
tokenizer.tar.gz}. This is a UNIX sed program. Run it with
sed -f. It needs an input file with one sentence per line. Apply
the tokenizer to the processed Alice corpus.
\item
Stemming. Download and compile the Porter stemmer from
\url{porter.c}. Run the stemmer on
Alice from the previous step. You will notice that it maps all words to lower case, and some words look funny.
\end{enumerate}
Question 5.1. Do not strip punctuations or otherwise change the
tokens out of the stemmer. How many word tokens and word types are
there?

Question 5.2. List the top 10 most frequent words (they can be
punctuations) and their counts.

Question 5.3. In Matlab, plot rank $r$ (x-axis) vs. count $f$ (y-axis)
for all words. Each word would be a dot in such a plot. In a second plot,
plot the same thing but use log scale on both axes.

Question 5.4. Assume the following relation: $f = ar^b$. Use Matlab
polyfit function to find a, b. Hint: take log on both sides.







\end{enumerate}


\end{document}
