\documentclass[color]{beamer}
\usetheme{default}
\usepackage{pgf,xcolor}
\usecolortheme{seahorse}
\setbeamersize{textmarginleft=0.5cm}
\setbeamersize{textmarginlright=0.5cm}
\setbeamercolor{verbcolor}{fg=blue,bg=lightgray}
% \usepackage{graphicx}
\usepackage{url}
\usepackage{natbib}
\usepackage{float}
% \usepackage{beamerthemesplit}
\title{Statistical Perspectives on Data Mining -- Summary of Modules}
\author{John Maindonald}

\begin{document}
\large
\frame{\titlepage}

\section{Overview}

\frame{
\frametitle{Overview of Lectures and Laboratories}

\begin{block}{Lectures and laboratories will}
\begin{itemize}
\item<1-> Introduce the R System\newline
R will be used for (almost) all computations.
\item<1-> Identify statistical ideas and issues that are important for
all data analysts.
\item<1-> Develop data analysis skills.
\item<1-> Give experience with data analysis strategies and methodologies.
\begin{itemize}
  \item<1-> Demonstrate classical statistical methodologies\newline
  (Linear models and extensions are especially important).
  \item<1-> Demonstrate ``data mining'' \& related ``new'' methodologies
\end{itemize}
\item<2-> Note connections and differences between the
different methodologies. Note advantages/disadvantages of each.
\end{itemize}
\end{block}
}

\frame{
\begin{block}{Key ``statistical'' concerns}
\begin{itemize}
\item Ask the ``right'' questions, i.e. ask questions that relate to 
intended/likely use(s) of results.
\item Different uses may require different analyses 
and/or different adapations of analysis results
\item Some desired uses may require different data!
\item These issues affect predictive generalization. Specifically:
\begin{itemize}
  \item What analysis, if any, will support the intended predictions?
  \item There are (at least) two models -- one for the data, and one
for the intended predictive use of the data.  They may or may not
be the same model!
\end{itemize}
\item Be aware of common traps, lest results are misleading or worthless.
\end{itemize}
\end{block}
}

\frame{
\frametitle{Course Overview -- Handouts}
\begin{itemize}
\item The R System -- An Introduction and Overview -- 88pp.\newline
\textbf{r-dm.pdf}
\item Statistical Perspectives on Data Mining (SPDM) -- 49pp.\newline
\textbf{statnotes.pdf}
\item 13 sets of laboratory exercises -- currently 71pp. in total.\newline
\textbf{r-exercises.pdf} (We will use a selection of these.)
\end{itemize}
}


\section{The R System}
\frame{
\frametitle{The R System:}
\framesubtitle{Laboratories I -- III}
\begin{itemize}
\item<1-> R is currently the environment of choice for 
\begin{itemize}
  \item<1-> specialists who are implementing new methodology
  \item<1-> highly trained professional data analysts.
\end{itemize}
\item<1-> It is designed for interactive data analysis: the next step
may depend on the previous result
\item<2-> It can be remarkably efficient, even though:
\begin{itemize}
  \item<2-> data resides (mostly) in memory
  \item<2-> it is an interpreted language (but one command may start a
lengthy computation)
\end{itemize}
\item<3-> New releases every few months bring improvements \& new features.
\end{itemize}
\vspace*{9pt}

See \textit{Rintro}; check out \url{http://cran.r-project.org}
}

\subsection{Simple R}

\begin{frame}[fragile]
\frametitle{Introduction to the R System}
\framesubtitle{Laboratories I -- III}

Mainly, read and work through \textit{Rintro}.\newline 
Overheads that introduce R are in preparation. For now:

\begin{block}{Simple R}

Type following \verb!>!, which is the command prompt.
\begin{beamercolorbox}{verbcolor}
\begin{verbatim}
> 2+2
[1] 4
>
\end{verbatim}
\end{beamercolorbox}
The \texttt{[1]} says, perhaps a little strangely,\newline 
``first requested element will follow''
\end{block}

\begin{block}{Demonstrations}
\begin{beamercolorbox}{verbcolor}
\begin{verbatim}
demo(graphics)   # Gives graphics demonstrations
demo()           # List all available demonstrations
\end{verbatim}
\end{beamercolorbox}
\end{block}
\end{frame}

\section{Statistical Ideas and Issues}
\frame{
\frametitle{Data Mining versus Statistics}
\framesubtitle{\textit{SPDM}, Section 1.1}
\begin{block}{What is data mining}
\begin{itemize}
\item Statistics at Scale and Speed (Daryl Pregibon)
\item More cynical definitions are possible!
 \begin{itemize}
  \item A combination of large databases and bad statistics
  \item Statistics plus marketing
 \end{itemize}
\item The next slide gives another perspective
\end{itemize}
\end{block}
\begin{block}{For statistics at Scale and Speed}
\begin{itemize}
\item Scale and speed make automation essential.
\item The skill lies in judging
 \begin{itemize}
  \item what to automate
  \item when to call on the skill of the human expert
  \item in using tabular and graphical summaries to call attention\newline
   to features of the data that might not otherwise be obvious
  \end{itemize}
\end{itemize}
\end{block}
}

\frame{
\frametitle{Statistical Ideas and Issues}
\framesubtitle{\textit{SPDM}, Section 1.1}
\begin{block}{Data Analysis -- What has changed in the past two decades?}
\begin{itemize}
\item Large datasets 
{\small (from automation; merging of databases)}
\item New types of data 
{\small (e.g., images, web pages, sound tracks)}
\item There are new vastly better data analysis tools\\[-4pt]
{\small (Computing and statistical theory have advanced hand in hand)}\\[6pt]
\item The new methods and software often allow:
  \begin{itemize}
   \item Analyses that better reflect the scientific questions
   \item Analyses that extract more of the information
   \item Meta-analysis across multiple studies, yielding information
not available from individual studies.
  \end{itemize}
\item Traps for the unwary are as serious as ever.\newline 
\textit{Watch this space!}
\end{itemize}
\textit{Data mining encompasses, broadly, the first three items above.}
\end{block}
}

\frame{
\frametitle{Statistical Ideas and Issues}
\framesubtitle{\textit{SPDM}, Sections 1.2 and 1.3}

\begin{block}{Questions}
\begin{itemize}
\item Why am I undertaking this investigation?
\item What is the intended use of results?\newline
{\color{violet}  What is the target population? Will results be used}\newline
{\color{violet} under the same conditions that generated the data\newline
 used to fit the model?  If not, are the differences fatal?}
\item What limitations in the data, from manner of collection\newline
or from incompleteness, may constrain the intended use?
{\color{violet} Issue: How were the data obtained (sampled)?}
\end{itemize}
\end{block}
Here, nothing has changed in recent decades!\newline
Such questions are as important as ever, even more\newline 
for observational than for experimental data.
}

\begin{frame}
\frametitle{Unequal subgroup weights -- traps}
\framesubtitle{Laboratory VIII; \textit{SPDM}, Section 3}

\begin{columns}[c]
\begin{column}{0.6\linewidth}
\includegraphics[scale=0.8]{/Users/johnm/coursenotes/dm/Art/over1.pdf}
\end{column}
\begin{column}{0.395\linewidth}
Does baclofen, following operation (additional to earlier painkiller), 
reduce pain?
\end{column}
\end{columns}
\vspace*{15pt}

\begin{columns}
\column{1.02\linewidth}
\begin{block}{Unequal subgroup numbers spell trouble!}
Subgroup numbers, shown below each point in the graph,
weight the overall averages when sex is ignored.

Baclofen: 15f to 3m, i.e. $\frac{15}{18}$ to $\frac{3}{18}$
\hspace*{4pt} (a little less than f average)
No baclofen: 7f to 9m, i.e. $\frac{7}{16}$ to $\frac{9}{16}$
\hspace*{4pt} ($\approx \frac{1}{2}$-way between m \& f)
\end{block}
\end{columns}
\end{frame}

\begin{frame}
\frametitle{Unequal subgroup weights -- a strategy}
\framesubtitle{Laboratory VIII; \textit{SPDM}, Section 3}
\begin{block}{Simple approach}
Calculate means for each subgroup separately.\\
Overall treatment effect is average of subgroup differences.\\
Effect of baclofen (reduction in pain score from time 0) is:
\begin{itemize}
 \item[] Females: 3.479 - 4.151 = -0.672 (-ve, $Rightarrow$ an increase)
 \item[] Males: 1.311 - 1.647 = -0.336
 \item[] Average of m and f = -0.5 $\times$ (0.672+0.336) = -0.504
\end{itemize}
\end{block}

\begin{block}{Fit a model that accounts for sex and baclofen effects}
$y$ = overall mean + sex effect + baclofen effect + interaction\newline
(At this point, we are not including an error term).
\end{block}
\begin{block}{Why specify a model?}
It makes assumptions explicit. More anon!\newline
\end{block} 
\end{frame}


\begin{frame}
\frametitle{When/how can models untangle subgroup effects?}
\framesubtitle{Laboratory VIII; \textit{SPDM}, Section 3}
\begin{block}{An Example -- Is moderate wine-drinking good for the heart?}
Many factors affect the risk of a heart attack -- genetic, lifestyle, diet, 
etc.
Modeling will be effective only if:
\begin{itemize}
\item Data include all factors that might inflence risk
\item The response is correctly modeled (for important variables, moderation
may be better than either extreme)
\item The model correctly accounts for interactions between factors.
\end{itemize}

Here, it may be impossible to disentangle subgroup effects\newline
(but several papers claim to demonstrate an effect.)
\end{block}
See Jackson et al. (2005). (reference in \textit{SPDM})
\end{frame}

\begin{frame}
\frametitle{Conditions for effective modeling}
\framesubtitle{Laboratory VIII; \textit{SPDM}, Section 3}
\begin{block}{Modeling is likely to be effective if}
\begin{itemize}
\item One or two factors have a large and dominant effect\newline
e.g., health effects of smoking or of exposure to asbestos
\item OR, a theoretical equation explains most variation, e.g.\newline
book weight = constant $\times$ height $\times$ breadth $\times$ thickness
\item OR (best of all), data are from a carefully designed experiment.
\end{itemize}
\end{block}
\begin{block}{Example -- hormone replacement therapy (HRT)}
\begin{itemize}
\item Initial data were observational (many factors, no good guidance
from theory)
\item More recently, there've been randomized controlled trials.
\end{itemize}
The experimental data gives definitive (\& different) results.
\end{block}
\end{frame}

\begin{frame}
\frametitle{Subgroup Effects -- tables of counts}
\framesubtitle{Laboratory VIII; \textit{SPDM}, Section 3}
\begin{block}{Example with contrived data}
\begin{tabular}{@{}lllllllll@{}}
\hline
&\multicolumn{2}{c}{Engineering} && \multicolumn{2}{c}{Sociology} && 
\multicolumn{2}{c@{}}{Total} \\[-4.5pt]
&\multicolumn{2}{c}{\hrulefill} && \multicolumn{2}{c}{\hrulefill} && 
\multicolumn{2}{c@{}}{\hrulefill} \\
& Female & Male  &  &Female & Male  &  &Female & Male  \\
\cline{2-3} \cline{5-6} \cline{8-9}
Admit  &10  &30  & &30  &15  &  &{\em 40} &{\em 45}   \\
Deny   &10  &30  & &10  & 5  &  &{\em 20} &{\em 35}   \\
\hline
\end{tabular}
\end{block}

\begin{block}{Admission rate calculations -- the weights}
\vspace*{6pt}

\begin{tabular}{ll}
Females: & 
$\frac{10}{20} \times \frac{20}{60} + \frac{30}{40} \times \frac{40}{60}$
\hspace*{12pt} [0.33 (Eng) : 0.66 (Soc)]\\[9pt]
Males: &
$\frac{30}{60} \times \frac{60}{80} + \frac{15}{20} \times \frac{20}{80}$
\hspace*{12pt} [0.75 (Eng) : 0.25 (Soc)]
\end{tabular}
\end{block}

\begin{block}{The Overall Rates}
\begin{itemize}
\item females ($\frac{2}{3}$): bias is towards the Sociology rate 
(0.75)
\item males ($\frac{45}{80}$): bias is towards the Engineering rate 
(0.5).
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{Models -- the two parts of a model}
\framesubtitle{\textit{SPDM}, Section 2, esp.\ Subsection 2.2, 
and Sections 14 and 15}
\begin{block}{Models have fixed and random parts. Common aims are:}

\begin{itemize}
\item Estimate the fixed part(s) 
\item Assess implications, for accuracy, of the random part(s).
\end{itemize}
\end{block}

\begin{block}{Three levels of sophistication in accounting for random error}
\begin{enumerate}
\item Ignore the random part (i.e., deterministic modeling)
\item Independently \& identically distributed (i.i.d.) errors
\item Assume a ``complex'' error structure\newline
(may be essential if there is $>$ 1 level of random variation)
\end{enumerate}
\end{block}

\begin{block}{How much sophistication is required?}
Use a simplified model if it serves the purpose.\\
(but careful modeling of the random part may
be needed to show that a simplified model is OK!)
\end{block}
\end{frame}

\begin{frame}
\frametitle{Models -- their importance for data analysts}
\framesubtitle{\textit{SPDM}, Section 2}
\begin{block}{There is not one model, but (at least) three!}
\begin{itemize}
   \item Data were sampled from a source population?\newline
{\color{violet} Was the sampling, effectively, random? Is bias an issue?}
   \item A model must be assumed, for purposes of analysis,\newline
e.g. Depression in lawn is proportional to roller weight.\newline
{\color{violet} Results perhaps apply to the lawn(s) in the sample.}
   \item How will results be used? Assuming use for prediction:\newline 
{\color{violet} Are target and source populations essentially the same?\\
Are predictions for the sampled lawn(s), or for new lawns?}
\end{itemize}
\end{block}

\begin{block}{Worst Case Scenario}

Data source and target population are totally different, e.g.\newline
Data on 2006 oil prices will give hopeless predictions for 2008!
\end{block}
\end{frame}

\begin{frame}
\frametitle{Models -- their importance for data analysts}
\framesubtitle{\textit{SPDM}, Section 2}
\begin{block}{Alternatively, the model might describe the total process}
\begin{itemize}
   \item For analysis a model is used that purports to describe\newline 
the processes (sampling \& other) that generated the data.\newline
e.g. Depression in lawn is proportional to roller weight.\newline
{\color{violet} It is now harder to say what is meant by bias.}
   \item For prediction, there must be a model that describes the\newline
processes that will apply when predictions are made.\newline
{\color{violet} Will the model that was used for analysis do the job?}
\end{itemize}
\end{block}

\begin{block}{Worst Case Scenario}
Is it possible to create a model for 2006 oil prices that will\newline
still be usable in 2008?
\end{block}
\end{frame}

\begin{frame}
\frametitle{Choosing and Comparing Models}
\begin{block}{Possible strategies}
\begin{itemize}
   \item Ensure that the model fits well (diagnostic checks, etc.)\newline
{\color{violet} Predictions should then be accurate and robust.\newline
Accurate and robust for what range of target populations?}
   \item Aim directly for accurate prediction (a data mining emphasis?)
{\color{violet} What population is in mind?}
\end{itemize}
\end{block}
\begin{block}{Commentary}
\begin{itemize}
\item Predictive accuracy will vary with the target population.
\item Often, good stats properties $\Rightarrow$ acceptable accuracy,\newline
for different possible different target populations.
\item Tuning predictions to a specific target can be risky.\newline
{\color{violet} Times change, populations also change!}
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{Assessment of Predictive Accuracy}
\framesubtitle{Laboratories X and XI will demonstrate cross-validation}

\begin{block}{Two Empirical Methods (both assume one model does all!)}

\begin{itemize}
\item The training/test approach can be used when source and\newline
target populations differ.\newline
{\color{violet} Requires a sample from target population.\newline
But what if, e.g., the target is a future population.\newline
e.g. train a model to detect computer intrusions.}
\item Cross-validation is popular and effective.\newline
{\color{violet} It assumes equivalence of source and target populations.}
\end{itemize}
\end{block}

Cross-validation uses multiple repeats of a training/test split,\newline
with a new training/test split at each repeat (see next slide).

\begin{block}{Theoretical Assessments}
\begin{itemize}
\item Classical statistical theory may give accuracy estimates.\newline
{\color{magenta} Such estimates may be highly sensitive to assumptions.}
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{Test/training Set and Cross-Validation}
\begin{block}{Steps}
\begin{itemize}
\item Split data into $k$ parts (below,$k$=4)
\item At the $i$th repeat or \textit{fold} ($i = 1, \ldots k$) use:\newline 
the $i$th part for \textit{testing}, the other $k$-1 parts for 
\textit{training}.
\item Combine the performance estimates from the $k$ folds.
\end{itemize}
\end{block}

\includegraphics[scale=0.8]{/Users/johnm/coursenotes/dm/Art/over0.pdf}
\end{frame}



\begin{frame}
\frametitle{Random Error in Models -- Examples}
\framesubtitle{\textit{SPDM}, Section 2}
\begin{block}{Example -- Distance fallen under gravity $d = 0.5 g t^2$}
\begin{itemize}
\item There is random error 
\begin{itemize}
\item $g$ is not quite constant as the object falls
\item there are fixed \& random effects from air resistance
\item there is measurement error
\end{itemize}
\item Often, the small random error can be ignored.
\end{itemize}
\end{block}

\begin{block}{Corn yields on Antigua (in the Caribbean)}
8 sites; 4 blocks per site. Sources of variation are:
\begin{itemize}
\item Within sites -- the relevant error for prediction to a new block
on one of the 8 sites
\item Between sites -- need to a/c for this, for prediction to a new block
on a new site.
\end{itemize}
\vspace*{6pt}

As complex error structures go, the error here is simple!
\end{block}
\end{frame}


\begin{frame}
\frametitle{Models -- the random part}
\framesubtitle{Laboratory IV; \textit{SPDM}, Part II}
\begin{block}{Distributions for the random part}
Note in particular:
\begin{itemize}
\item the normal distribution
\item the empirical data distribution\newline
(treat the sample data as a microcosm of the population)
\item the distributions covered in laboratories III and IV are often used
as building blocks for ``complex'' distributions.
\end{itemize}
\end{block}
\begin{block}{When/where are ``complex'' distributions important?}
\begin{itemize}
\item There may be $>$ 1 relevant source of random variation.
\item Time series.
\item Other examples abound, but this will do for now.
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{The Sampling Distribution of the Mean}
\framesubtitle{Laboratory V; \textit{SPDM}, Part II}
\includegraphics{/Users/johnm/coursenotes/dm/Art/over2.pdf}

Numbers (1, 4, 16) in the legend are sample sizes.\newline
The right panel used heights of female Adelaide U students.
\end{frame}

\begin{frame}
\frametitle{Sampling Distributions -- their Use for Inference}
\framesubtitle{Laboratory V; \textit{SPDM}, Part II}
\begin{block}{The sampling distribution of the mean}
 There is (usually) a single sample of data, \& a single mean.\newline
 The sampling distribution estimates the distribution of the mean
under repeated sampling.  
\end{block}

\begin{block}{Estimating the sampling distribution}
Alternatives are:
\begin{itemize}
\item Use theory, usually making i.i.d.\ assumptions\newline
Inference is then based on the $t$-distribution;
\item Base inference on the empirically estimated distribution 
under repeated resampling (the bootstrap method).\newline
[Most useful for statistics other than the mean.]
\end{itemize}
\end{block}

\begin{block}{Further discussion is deferred till later}
\end{block}
\end{frame}

\begin{frame}
\frametitle{Linear Models - a simple example}
\framesubtitle{Laboratory VI; \textit{SPDM}, Section 9}

Straightline model:
$y_i = b x_i +e_i \;\;(i = 1, 2, \ldots n)$
\vspace*{6pt}

\begin{columns}[c]
\begin{column}{0.65\linewidth}
\includegraphics{/Users/johnm/coursenotes/dm/Art/over3.pdf}
\end{column}
\begin{column}{0.01\linewidth}
\end{column}
\begin{column}{0.35\linewidth}
\begin{block}{Model in the graph:}
depression = \newline
$b$ $\times$ weight + error
\end{block}
\vspace*{9pt}

The line has been forced through the origin. Is this desirable?

\begin{block}{Fitting the model}
Typically, use least squares, i.e., choose $b$ to minimize $\sum e_i^2$.
\end{block}
\end{column}
\end{columns}
\end{frame}

\begin{frame}
\frametitle{Linear Models - Why?}
\framesubtitle{Laboratories VI and VII; \textit{SPDM}, Sections 9--13}
\begin{block}{Start discussion of models with linear models because:}
\begin{itemize}
\item the statistical theory is well understood;
\item linear models are widely used;
\item linear models are simple in concept, and easy to fit;
\item ideas that will be explained in this context are important\newline
  also for other models, including ``data mining'' models;
\item non-linear models are often built together, in part,\newline
from linear models.
\end{itemize}
\end{block}
\end{frame}


\begin{frame}
\frametitle{Multiple Linear Regression}
\framesubtitle{Laboratory VII; \textit{SPDM}, Sections 10--13}
\begin{block}{Multiple explanatory variables}
\begin{itemize}
\item Adding more explanatory variables is easy, e.g.\newline
  $\log(\mbox{bookweight}) = a + b_1 \times \log(\mbox{area of cover})
  + b_2 \times \log(\mbox{thickness}) + \mbox{error}$
\item Knowing what results mean can be hard or impossible:
\begin{itemize}
\item One variable can mask or modify the effect of another.
\item It is often impossible to be sure that all relevant effects\newline
have been accounted for.
\item There may be biases in data collection.
\item Some variables may be so inaccurately measured that their\newline
effects are undetectable.
\end{itemize}
\end{itemize}
\end{block}

Practical use of multiple regression, to give results that\newline
carry conviction, can be (in many contexts, is) challenging! 
\end{frame}

\begin{frame}
\frametitle{The Wide Reach of Linear Models}
\framesubtitle{Laboratory VII; \textit{SPDM}, Sections 10--13}
\begin{block}{Linear models can do much, much more!}
\begin{itemize}
\item Qualitative effects can be modeled
\item Effects can be smooth curves (use regression splines)
\end{itemize}
\end{block}
\vspace*{-2pt}

These extensions use the same linear modeling theory.

\begin{block}{Linear components in non-linear models}
Non-linear models may have ``linear'' component(s).\newline
Model-fitting algorithms can take advantage of this.
\end{block}
\end{frame}


\begin{frame}
\frametitle{Models for Classification \& Discrimination}
\framesubtitle{Laboratories X and XI -- insert a graph that shows a 
2-dim discriminant line}

\begin{block}{There are many different types of discriminant model}
\begin{itemize}
\item ``Classical'' linear discriminants (NB: ``non-linear'' uses)
\item ``Algorithmic' (e.g., tree-based, neural nets)\newline
 (Data miners have often preferred algorithmic models.)
\end{itemize}
\end{block}

\begin{block}{Comparison}
\begin{itemize}
\item Classical models have a well-developed statistical theory.\newline
(accuracy estimates may be approximate, but adequate?).
\item Classical models may allow mechanistic interpretations.
\item Algorithmic models may give more directly usable results.\newline
(e.g., a simple diagnostic rule)
\item Non-classical may outperform classical models when there\newline 
is a complex pattern of effects, and data are extensive.
\end{itemize}
\end{block}
\vspace*{12pt}
\end{frame}

\end{document}
