{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "_You are currently looking at **version 1.0** of this notebook. To download notebooks and datafiles, as well as get help on Jupyter notebooks in the Coursera platform, visit the [Jupyter Notebook FAQ](https://www.coursera.org/learn/python-machine-learning/resources/bANLa) course resource._\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Applied Machine Learning: Module 3 (Evaluation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation for Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preamble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import load_digits\n",
    "\n",
    "dataset = load_digits()\n",
    "X, y = dataset.data, dataset.target\n",
    "\n",
    "for class_name, class_count in zip(dataset.target_names, np.bincount(dataset.target)):\n",
    "    print(class_name,class_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Creating a dataset with imbalanced binary classes:  \n",
    "# Negative class (0) is 'not digit 1' \n",
    "# Positive class (1) is 'digit 1'\n",
    "y_binary_imbalanced = y.copy()\n",
    "y_binary_imbalanced[y_binary_imbalanced != 1] = 0\n",
    "\n",
    "print('Original labels:\\t', y[1:30])\n",
    "print('New binary labels:\\t', y_binary_imbalanced[1:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "np.bincount(y_binary_imbalanced)    # Negative class (0) is the most frequent class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
    "\n",
    "# Accuracy of Support Vector Machine classifier\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "svm = SVC(kernel='rbf', C=1).fit(X_train, y_train)\n",
    "svm.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dummy Classifiers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "DummyClassifier is a classifier that makes predictions using simple rules, which can be useful as a baseline for comparison against actual classifiers, especially with imbalanced classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.dummy import DummyClassifier\n",
    "\n",
    "# Negative class (0) is most frequent\n",
    "dummy_majority = DummyClassifier(strategy = 'most_frequent').fit(X_train, y_train)\n",
    "# Therefore the dummy 'most_frequent' classifier always predicts class 0\n",
    "y_dummy_predictions = dummy_majority.predict(X_test)\n",
    "\n",
    "y_dummy_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dummy_majority.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "svm = SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
    "svm.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Confusion matrices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Binary (two-class) confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# Negative class (0) is most frequent\n",
    "dummy_majority = DummyClassifier(strategy = 'most_frequent').fit(X_train, y_train)\n",
    "y_majority_predicted = dummy_majority.predict(X_test)\n",
    "confusion = confusion_matrix(y_test, y_majority_predicted)\n",
    "\n",
    "print('Most frequent class (dummy classifier)\\n', confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# produces random predictions w/ same class proportion as training set\n",
    "dummy_classprop = DummyClassifier(strategy='stratified').fit(X_train, y_train)\n",
    "y_classprop_predicted = dummy_classprop.predict(X_test)\n",
    "confusion = confusion_matrix(y_test, y_classprop_predicted)\n",
    "\n",
    "print('Random class-proportional prediction (dummy classifier)\\n', confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "svm = SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
    "svm_predicted = svm.predict(X_test)\n",
    "confusion = confusion_matrix(y_test, svm_predicted)\n",
    "\n",
    "print('Support vector machine classifier (linear kernel, C=1)\\n', confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression().fit(X_train, y_train)\n",
    "lr_predicted = lr.predict(X_test)\n",
    "confusion = confusion_matrix(y_test, lr_predicted)\n",
    "\n",
    "print('Logistic regression classifier (default settings)\\n', confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "dt = DecisionTreeClassifier(max_depth=2).fit(X_train, y_train)\n",
    "tree_predicted = dt.predict(X_test)\n",
    "confusion = confusion_matrix(y_test, tree_predicted)\n",
    "\n",
    "print('Decision tree classifier (max_depth = 2)\\n', confusion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation metrics for binary classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "# Accuracy = TP + TN / (TP + TN + FP + FN)\n",
    "# Precision = TP / (TP + FP)\n",
    "# Recall = TP / (TP + FN)  Also known as sensitivity, or True Positive Rate\n",
    "# F1 = 2 * Precision * Recall / (Precision + Recall) \n",
    "print('Accuracy: {:.2f}'.format(accuracy_score(y_test, tree_predicted)))\n",
    "print('Precision: {:.2f}'.format(precision_score(y_test, tree_predicted)))\n",
    "print('Recall: {:.2f}'.format(recall_score(y_test, tree_predicted)))\n",
    "print('F1: {:.2f}'.format(f1_score(y_test, tree_predicted)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Combined report with all above metrics\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_test, tree_predicted, target_names=['not 1', '1']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print('Random class-proportional (dummy)\\n', \n",
    "      classification_report(y_test, y_classprop_predicted, target_names=['not 1', '1']))\n",
    "print('SVM\\n', \n",
    "      classification_report(y_test, svm_predicted, target_names = ['not 1', '1']))\n",
    "print('Logistic regression\\n', \n",
    "      classification_report(y_test, lr_predicted, target_names = ['not 1', '1']))\n",
    "print('Decision tree\\n', \n",
    "      classification_report(y_test, tree_predicted, target_names = ['not 1', '1']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
    "y_scores_lr = lr.fit(X_train, y_train).decision_function(X_test)\n",
    "y_score_list = list(zip(y_test[0:20], y_scores_lr[0:20]))\n",
    "\n",
    "# show the decision_function scores for first 20 instances\n",
    "y_score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
    "y_proba_lr = lr.fit(X_train, y_train).predict_proba(X_test)\n",
    "y_proba_list = list(zip(y_test[0:20], y_proba_lr[0:20,1]))\n",
    "\n",
    "# show the probability of positive class for first 20 instances\n",
    "y_proba_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Precision-recall curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, y_scores_lr)\n",
    "closest_zero = np.argmin(np.abs(thresholds))\n",
    "closest_zero_p = precision[closest_zero]\n",
    "closest_zero_r = recall[closest_zero]\n",
    "\n",
    "plt.figure()\n",
    "plt.xlim([0.0, 1.01])\n",
    "plt.ylim([0.0, 1.01])\n",
    "plt.plot(precision, recall, label='Precision-Recall Curve')\n",
    "plt.plot(closest_zero_p, closest_zero_r, 'o', markersize = 12, fillstyle = 'none', c='r', mew=3)\n",
    "plt.xlabel('Precision', fontsize=16)\n",
    "plt.ylabel('Recall', fontsize=16)\n",
    "plt.axes().set_aspect('equal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROC curves, Area-Under-Curve (AUC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
    "\n",
    "y_score_lr = lr.fit(X_train, y_train).decision_function(X_test)\n",
    "fpr_lr, tpr_lr, _ = roc_curve(y_test, y_score_lr)\n",
    "roc_auc_lr = auc(fpr_lr, tpr_lr)\n",
    "\n",
    "plt.figure()\n",
    "plt.xlim([-0.01, 1.00])\n",
    "plt.ylim([-0.01, 1.01])\n",
    "plt.plot(fpr_lr, tpr_lr, lw=3, label='LogRegr ROC curve (area = {:0.2f})'.format(roc_auc_lr))\n",
    "plt.xlabel('False Positive Rate', fontsize=16)\n",
    "plt.ylabel('True Positive Rate', fontsize=16)\n",
    "plt.title('ROC curve (1-of-10 digits classifier)', fontsize=16)\n",
    "plt.legend(loc='lower right', fontsize=13)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n",
    "plt.axes().set_aspect('equal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from matplotlib import cm\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
    "\n",
    "plt.figure()\n",
    "plt.xlim([-0.01, 1.00])\n",
    "plt.ylim([-0.01, 1.01])\n",
    "for g in [0.01, 0.1, 0.20, 1]:\n",
    "    svm = SVC(gamma=g).fit(X_train, y_train)\n",
    "    y_score_svm = svm.decision_function(X_test)\n",
    "    fpr_svm, tpr_svm, _ = roc_curve(y_test, y_score_svm)\n",
    "    roc_auc_svm = auc(fpr_svm, tpr_svm)\n",
    "    accuracy_svm = svm.score(X_test, y_test)\n",
    "    print(\"gamma = {:.2f}  accuracy = {:.2f}   AUC = {:.2f}\".format(g, accuracy_svm, \n",
    "                                                                    roc_auc_svm))\n",
    "    plt.plot(fpr_svm, tpr_svm, lw=3, alpha=0.7, \n",
    "             label='SVM (gamma = {:0.2f}, area = {:0.2f})'.format(g, roc_auc_svm))\n",
    "\n",
    "plt.xlabel('False Positive Rate', fontsize=16)\n",
    "plt.ylabel('True Positive Rate (Recall)', fontsize=16)\n",
    "plt.plot([0, 1], [0, 1], color='k', lw=0.5, linestyle='--')\n",
    "plt.legend(loc=\"lower right\", fontsize=11)\n",
    "plt.title('ROC curve: (1-of-10 digits classifier)', fontsize=16)\n",
    "plt.axes().set_aspect('equal')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation measures for multi-class classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Multi-class confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dataset = load_digits()\n",
    "X, y = dataset.data, dataset.target\n",
    "X_train_mc, X_test_mc, y_train_mc, y_test_mc = train_test_split(X, y, random_state=0)\n",
    "\n",
    "\n",
    "svm = SVC(kernel = 'linear').fit(X_train_mc, y_train_mc)\n",
    "svm_predicted_mc = svm.predict(X_test_mc)\n",
    "confusion_mc = confusion_matrix(y_test_mc, svm_predicted_mc)\n",
    "df_cm = pd.DataFrame(confusion_mc, \n",
    "                     index = [i for i in range(0,10)], columns = [i for i in range(0,10)])\n",
    "\n",
    "plt.figure(figsize=(5.5,4))\n",
    "sns.heatmap(df_cm, annot=True)\n",
    "plt.title('SVM Linear Kernel \\nAccuracy:{0:.3f}'.format(accuracy_score(y_test_mc, \n",
    "                                                                       svm_predicted_mc)))\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "\n",
    "\n",
    "svm = SVC(kernel = 'rbf').fit(X_train_mc, y_train_mc)\n",
    "svm_predicted_mc = svm.predict(X_test_mc)\n",
    "confusion_mc = confusion_matrix(y_test_mc, svm_predicted_mc)\n",
    "df_cm = pd.DataFrame(confusion_mc, index = [i for i in range(0,10)],\n",
    "                  columns = [i for i in range(0,10)])\n",
    "\n",
    "plt.figure(figsize = (5.5,4))\n",
    "sns.heatmap(df_cm, annot=True)\n",
    "plt.title('SVM RBF Kernel \\nAccuracy:{0:.3f}'.format(accuracy_score(y_test_mc, \n",
    "                                                                    svm_predicted_mc)))\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Multi-class classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(classification_report(y_test_mc, svm_predicted_mc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Micro- vs. macro-averaged metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('Micro-averaged precision = {:.2f} (treat instances equally)'\n",
    "      .format(precision_score(y_test_mc, svm_predicted_mc, average = 'micro')))\n",
    "print('Macro-averaged precision = {:.2f} (treat classes equally)'\n",
    "      .format(precision_score(y_test_mc, svm_predicted_mc, average = 'macro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('Micro-averaged f1 = {:.2f} (treat instances equally)'\n",
    "      .format(f1_score(y_test_mc, svm_predicted_mc, average = 'micro')))\n",
    "print('Macro-averaged f1 = {:.2f} (treat classes equally)'\n",
    "      .format(f1_score(y_test_mc, svm_predicted_mc, average = 'macro')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression evaluation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import datasets\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.dummy import DummyRegressor\n",
    "\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "X = diabetes.data[:, None, 6]\n",
    "y = diabetes.target\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "\n",
    "lm = LinearRegression().fit(X_train, y_train)\n",
    "lm_dummy_mean = DummyRegressor(strategy = 'mean').fit(X_train, y_train)\n",
    "\n",
    "y_predict = lm.predict(X_test)\n",
    "y_predict_dummy_mean = lm_dummy_mean.predict(X_test)\n",
    "\n",
    "print('Linear model, coefficients: ', lm.coef_)\n",
    "print(\"Mean squared error (dummy): {:.2f}\".format(mean_squared_error(y_test, \n",
    "                                                                     y_predict_dummy_mean)))\n",
    "print(\"Mean squared error (linear model): {:.2f}\".format(mean_squared_error(y_test, y_predict)))\n",
    "print(\"r2_score (dummy): {:.2f}\".format(r2_score(y_test, y_predict_dummy_mean)))\n",
    "print(\"r2_score (linear model): {:.2f}\".format(r2_score(y_test, y_predict)))\n",
    "\n",
    "# Plot outputs\n",
    "plt.scatter(X_test, y_test,  color='black')\n",
    "plt.plot(X_test, y_predict, color='green', linewidth=2)\n",
    "plt.plot(X_test, y_predict_dummy_mean, color='red', linestyle = 'dashed', \n",
    "         linewidth=2, label = 'dummy')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model selection using evaluation metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cross-validation example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "dataset = load_digits()\n",
    "# again, making this a binary problem with 'digit 1' as positive class \n",
    "# and 'not 1' as negative class\n",
    "X, y = dataset.data, dataset.target == 1\n",
    "clf = SVC(kernel='linear', C=1)\n",
    "\n",
    "# accuracy is the default scoring metric\n",
    "print('Cross-validation (accuracy)', cross_val_score(clf, X, y, cv=5))\n",
    "# use AUC as scoring metric\n",
    "print('Cross-validation (AUC)', cross_val_score(clf, X, y, cv=5, scoring = 'roc_auc'))\n",
    "# use recall as scoring metric\n",
    "print('Cross-validation (recall)', cross_val_score(clf, X, y, cv=5, scoring = 'recall'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Grid search example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "dataset = load_digits()\n",
    "X, y = dataset.data, dataset.target == 1\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "\n",
    "clf = SVC(kernel='rbf')\n",
    "grid_values = {'gamma': [0.001, 0.01, 0.05, 0.1, 1, 10, 100]}\n",
    "\n",
    "# default metric to optimize over grid parameters: accuracy\n",
    "grid_clf_acc = GridSearchCV(clf, param_grid = grid_values)\n",
    "grid_clf_acc.fit(X_train, y_train)\n",
    "y_decision_fn_scores_acc = grid_clf_acc.decision_function(X_test) \n",
    "\n",
    "print('Grid best parameter (max. accuracy): ', grid_clf_acc.best_params_)\n",
    "print('Grid best score (accuracy): ', grid_clf_acc.best_score_)\n",
    "\n",
    "# alternative metric to optimize over grid parameters: AUC\n",
    "grid_clf_auc = GridSearchCV(clf, param_grid = grid_values, scoring = 'roc_auc')\n",
    "grid_clf_auc.fit(X_train, y_train)\n",
    "y_decision_fn_scores_auc = grid_clf_auc.decision_function(X_test) \n",
    "\n",
    "print('Test set AUC: ', roc_auc_score(y_test, y_decision_fn_scores_auc))\n",
    "print('Grid best parameter (max. AUC): ', grid_clf_auc.best_params_)\n",
    "print('Grid best score (AUC): ', grid_clf_auc.best_score_)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation metrics supported for model selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.scorer import SCORERS\n",
    "\n",
    "print(sorted(list(SCORERS.keys())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Two-feature classification example using the digits dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Optimizing a classifier using different evaluation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "from sklearn.model_selection import train_test_split\n",
    "from adspy_shared_utilities import plot_class_regions_for_classifier_subplot\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "\n",
    "dataset = load_digits()\n",
    "X, y = dataset.data, dataset.target == 1\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "\n",
    "# Create a two-feature input vector matching the example plot above\n",
    "# We jitter the points (add a small amount of random noise) in case there are areas\n",
    "# in feature space where many instances have the same features.\n",
    "jitter_delta = 0.25\n",
    "X_twovar_train = X_train[:,[20,59]]+ np.random.rand(X_train.shape[0], 2) - jitter_delta\n",
    "X_twovar_test  = X_test[:,[20,59]] + np.random.rand(X_test.shape[0], 2) - jitter_delta\n",
    "\n",
    "clf = SVC(kernel = 'linear').fit(X_twovar_train, y_train)\n",
    "grid_values = {'class_weight':['balanced', {1:2},{1:3},{1:4},{1:5},{1:10},{1:20},{1:50}]}\n",
    "plt.figure(figsize=(9,6))\n",
    "for i, eval_metric in enumerate(('precision','recall', 'f1','roc_auc')):\n",
    "    grid_clf_custom = GridSearchCV(clf, param_grid=grid_values, scoring=eval_metric)\n",
    "    grid_clf_custom.fit(X_twovar_train, y_train)\n",
    "    print('Grid best parameter (max. {0}): {1}'\n",
    "          .format(eval_metric, grid_clf_custom.best_params_))\n",
    "    print('Grid best score ({0}): {1}'\n",
    "          .format(eval_metric, grid_clf_custom.best_score_))\n",
    "    plt.subplots_adjust(wspace=0.3, hspace=0.3)\n",
    "    plot_class_regions_for_classifier_subplot(grid_clf_custom, X_twovar_test, y_test, None,\n",
    "                                             None, None,  plt.subplot(2, 2, i+1))\n",
    "    \n",
    "    plt.title(eval_metric+'-oriented SVC')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Precision-recall curve for the default SVC classifier (with balanced class weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from adspy_shared_utilities import plot_class_regions_for_classifier\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "dataset = load_digits()\n",
    "X, y = dataset.data, dataset.target == 1\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "\n",
    "# create a two-feature input vector matching the example plot above\n",
    "jitter_delta = 0.25\n",
    "X_twovar_train = X_train[:,[20,59]]+ np.random.rand(X_train.shape[0], 2) - jitter_delta\n",
    "X_twovar_test  = X_test[:,[20,59]] + np.random.rand(X_test.shape[0], 2) - jitter_delta\n",
    "\n",
    "clf = SVC(kernel='linear', class_weight='balanced').fit(X_twovar_train, y_train)\n",
    "\n",
    "y_scores = clf.decision_function(X_twovar_test)\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, y_scores)\n",
    "closest_zero = np.argmin(np.abs(thresholds))\n",
    "closest_zero_p = precision[closest_zero]\n",
    "closest_zero_r = recall[closest_zero]\n",
    "\n",
    "plot_class_regions_for_classifier(clf, X_twovar_test, y_test)\n",
    "plt.title(\"SVC, class_weight = 'balanced', optimized for accuracy\")\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.xlim([0.0, 1.01])\n",
    "plt.ylim([0.0, 1.01])\n",
    "plt.title (\"Precision-recall curve: SVC, class_weight = 'balanced'\")\n",
    "plt.plot(precision, recall, label = 'Precision-Recall Curve')\n",
    "plt.plot(closest_zero_p, closest_zero_r, 'o', markersize=12, fillstyle='none', c='r', mew=3)\n",
    "plt.xlabel('Precision', fontsize=16)\n",
    "plt.ylabel('Recall', fontsize=16)\n",
    "plt.axes().set_aspect('equal')\n",
    "plt.show()\n",
    "print('At zero threshold, precision: {:.2f}, recall: {:.2f}'\n",
    "      .format(closest_zero_p, closest_zero_r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}