Format: application/octet-stream
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "metadata": {},
6 "source": [
7 "---\n",
8 "\n",
9 "_You are currently looking at **version 1.0** of this notebook. To download notebooks and datafiles, as well as get help on Jupyter notebooks in the Coursera platform, visit the [Jupyter Notebook FAQ](https://www.coursera.org/learn/python-machine-learning/resources/bANLa) course resource._\n",
10 "\n",
11 "---"
12 ]
13 },
14 {
15 "cell_type": "markdown",
16 "metadata": {
17 "collapsed": true
18 },
19 "source": [
20 "# Applied Machine Learning: Module 3 (Evaluation)"
21 ]
22 },
23 {
24 "cell_type": "markdown",
25 "metadata": {},
26 "source": [
27 "## Evaluation for Classification"
28 ]
29 },
30 {
31 "cell_type": "markdown",
32 "metadata": {},
33 "source": [
34 "### Preamble"
35 ]
36 },
37 {
38 "cell_type": "code",
39 "execution_count": null,
40 "metadata": {
41 "collapsed": false
42 },
43 "outputs": [],
44 "source": [
45 "%matplotlib notebook\n",
46 "import numpy as np\n",
47 "import pandas as pd\n",
48 "import seaborn as sns\n",
49 "import matplotlib.pyplot as plt\n",
50 "from sklearn.model_selection import train_test_split\n",
51 "from sklearn.datasets import load_digits\n",
52 "\n",
53 "dataset = load_digits()\n",
54 "X, y = dataset.data, dataset.target\n",
55 "\n",
56 "for class_name, class_count in zip(dataset.target_names, np.bincount(dataset.target)):\n",
57 " print(class_name,class_count)"
58 ]
59 },
60 {
61 "cell_type": "code",
62 "execution_count": null,
63 "metadata": {
64 "collapsed": false
65 },
66 "outputs": [],
67 "source": [
68 "# Creating a dataset with imbalanced binary classes: \n",
69 "# Negative class (0) is 'not digit 1' \n",
70 "# Positive class (1) is 'digit 1'\n",
71 "y_binary_imbalanced = y.copy()\n",
72 "y_binary_imbalanced[y_binary_imbalanced != 1] = 0\n",
73 "\n",
74 "print('Original labels:\\t', y[1:30])\n",
75 "print('New binary labels:\\t', y_binary_imbalanced[1:30])"
76 ]
77 },
78 {
79 "cell_type": "code",
80 "execution_count": null,
81 "metadata": {
82 "collapsed": false,
83 "scrolled": true
84 },
85 "outputs": [],
86 "source": [
87 "np.bincount(y_binary_imbalanced) # Negative class (0) is the most frequent class"
88 ]
89 },
90 {
91 "cell_type": "code",
92 "execution_count": null,
93 "metadata": {
94 "collapsed": false
95 },
96 "outputs": [],
97 "source": [
98 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
99 "\n",
100 "# Accuracy of Support Vector Machine classifier\n",
101 "from sklearn.svm import SVC\n",
102 "\n",
103 "svm = SVC(kernel='rbf', C=1).fit(X_train, y_train)\n",
104 "svm.score(X_test, y_test)"
105 ]
106 },
107 {
108 "cell_type": "markdown",
109 "metadata": {},
110 "source": [
111 "### Dummy Classifiers"
112 ]
113 },
114 {
115 "cell_type": "markdown",
116 "metadata": {
117 "collapsed": true
118 },
119 "source": [
120 "DummyClassifier is a classifier that makes predictions using simple rules, which can be useful as a baseline for comparison against actual classifiers, especially with imbalanced classes."
121 ]
122 },
123 {
124 "cell_type": "code",
125 "execution_count": null,
126 "metadata": {
127 "collapsed": false
128 },
129 "outputs": [],
130 "source": [
131 "from sklearn.dummy import DummyClassifier\n",
132 "\n",
133 "# Negative class (0) is most frequent\n",
134 "dummy_majority = DummyClassifier(strategy = 'most_frequent').fit(X_train, y_train)\n",
135 "# Therefore the dummy 'most_frequent' classifier always predicts class 0\n",
136 "y_dummy_predictions = dummy_majority.predict(X_test)\n",
137 "\n",
138 "y_dummy_predictions"
139 ]
140 },
141 {
142 "cell_type": "code",
143 "execution_count": null,
144 "metadata": {
145 "collapsed": false
146 },
147 "outputs": [],
148 "source": [
149 "dummy_majority.score(X_test, y_test)"
150 ]
151 },
152 {
153 "cell_type": "code",
154 "execution_count": null,
155 "metadata": {
156 "collapsed": false
157 },
158 "outputs": [],
159 "source": [
160 "svm = SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
161 "svm.score(X_test, y_test)"
162 ]
163 },
164 {
165 "cell_type": "markdown",
166 "metadata": {},
167 "source": [
168 "### Confusion matrices"
169 ]
170 },
171 {
172 "cell_type": "markdown",
173 "metadata": {},
174 "source": [
175 "#### Binary (two-class) confusion matrix"
176 ]
177 },
178 {
179 "cell_type": "code",
180 "execution_count": null,
181 "metadata": {
182 "collapsed": false
183 },
184 "outputs": [],
185 "source": [
186 "from sklearn.metrics import confusion_matrix\n",
187 "\n",
188 "# Negative class (0) is most frequent\n",
189 "dummy_majority = DummyClassifier(strategy = 'most_frequent').fit(X_train, y_train)\n",
190 "y_majority_predicted = dummy_majority.predict(X_test)\n",
191 "confusion = confusion_matrix(y_test, y_majority_predicted)\n",
192 "\n",
193 "print('Most frequent class (dummy classifier)\\n', confusion)"
194 ]
195 },
196 {
197 "cell_type": "code",
198 "execution_count": null,
199 "metadata": {
200 "collapsed": false
201 },
202 "outputs": [],
203 "source": [
204 "# produces random predictions w/ same class proportion as training set\n",
205 "dummy_classprop = DummyClassifier(strategy='stratified').fit(X_train, y_train)\n",
206 "y_classprop_predicted = dummy_classprop.predict(X_test)\n",
207 "confusion = confusion_matrix(y_test, y_classprop_predicted)\n",
208 "\n",
209 "print('Random class-proportional prediction (dummy classifier)\\n', confusion)"
210 ]
211 },
212 {
213 "cell_type": "code",
214 "execution_count": null,
215 "metadata": {
216 "collapsed": false,
217 "scrolled": true
218 },
219 "outputs": [],
220 "source": [
221 "svm = SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
222 "svm_predicted = svm.predict(X_test)\n",
223 "confusion = confusion_matrix(y_test, svm_predicted)\n",
224 "\n",
225 "print('Support vector machine classifier (linear kernel, C=1)\\n', confusion)"
226 ]
227 },
228 {
229 "cell_type": "code",
230 "execution_count": null,
231 "metadata": {
232 "collapsed": false
233 },
234 "outputs": [],
235 "source": [
236 "from sklearn.linear_model import LogisticRegression\n",
237 "\n",
238 "lr = LogisticRegression().fit(X_train, y_train)\n",
239 "lr_predicted = lr.predict(X_test)\n",
240 "confusion = confusion_matrix(y_test, lr_predicted)\n",
241 "\n",
242 "print('Logistic regression classifier (default settings)\\n', confusion)"
243 ]
244 },
245 {
246 "cell_type": "code",
247 "execution_count": null,
248 "metadata": {
249 "collapsed": false
250 },
251 "outputs": [],
252 "source": [
253 "from sklearn.tree import DecisionTreeClassifier\n",
254 "\n",
255 "dt = DecisionTreeClassifier(max_depth=2).fit(X_train, y_train)\n",
256 "tree_predicted = dt.predict(X_test)\n",
257 "confusion = confusion_matrix(y_test, tree_predicted)\n",
258 "\n",
259 "print('Decision tree classifier (max_depth = 2)\\n', confusion)"
260 ]
261 },
262 {
263 "cell_type": "markdown",
264 "metadata": {},
265 "source": [
266 "### Evaluation metrics for binary classification"
267 ]
268 },
269 {
270 "cell_type": "code",
271 "execution_count": null,
272 "metadata": {
273 "collapsed": false
274 },
275 "outputs": [],
276 "source": [
277 "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
278 "# Accuracy = TP + TN / (TP + TN + FP + FN)\n",
279 "# Precision = TP / (TP + FP)\n",
280 "# Recall = TP / (TP + FN) Also known as sensitivity, or True Positive Rate\n",
281 "# F1 = 2 * Precision * Recall / (Precision + Recall) \n",
282 "print('Accuracy: {:.2f}'.format(accuracy_score(y_test, tree_predicted)))\n",
283 "print('Precision: {:.2f}'.format(precision_score(y_test, tree_predicted)))\n",
284 "print('Recall: {:.2f}'.format(recall_score(y_test, tree_predicted)))\n",
285 "print('F1: {:.2f}'.format(f1_score(y_test, tree_predicted)))"
286 ]
287 },
288 {
289 "cell_type": "code",
290 "execution_count": null,
291 "metadata": {
292 "collapsed": false
293 },
294 "outputs": [],
295 "source": [
296 "# Combined report with all above metrics\n",
297 "from sklearn.metrics import classification_report\n",
298 "\n",
299 "print(classification_report(y_test, tree_predicted, target_names=['not 1', '1']))"
300 ]
301 },
302 {
303 "cell_type": "code",
304 "execution_count": null,
305 "metadata": {
306 "collapsed": false,
307 "scrolled": false
308 },
309 "outputs": [],
310 "source": [
311 "print('Random class-proportional (dummy)\\n', \n",
312 " classification_report(y_test, y_classprop_predicted, target_names=['not 1', '1']))\n",
313 "print('SVM\\n', \n",
314 " classification_report(y_test, svm_predicted, target_names = ['not 1', '1']))\n",
315 "print('Logistic regression\\n', \n",
316 " classification_report(y_test, lr_predicted, target_names = ['not 1', '1']))\n",
317 "print('Decision tree\\n', \n",
318 " classification_report(y_test, tree_predicted, target_names = ['not 1', '1']))"
319 ]
320 },
321 {
322 "cell_type": "markdown",
323 "metadata": {},
324 "source": [
325 "### Decision functions"
326 ]
327 },
328 {
329 "cell_type": "code",
330 "execution_count": null,
331 "metadata": {
332 "collapsed": false
333 },
334 "outputs": [],
335 "source": [
336 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
337 "y_scores_lr = lr.fit(X_train, y_train).decision_function(X_test)\n",
338 "y_score_list = list(zip(y_test[0:20], y_scores_lr[0:20]))\n",
339 "\n",
340 "# show the decision_function scores for first 20 instances\n",
341 "y_score_list"
342 ]
343 },
344 {
345 "cell_type": "code",
346 "execution_count": null,
347 "metadata": {
348 "collapsed": false
349 },
350 "outputs": [],
351 "source": [
352 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
353 "y_proba_lr = lr.fit(X_train, y_train).predict_proba(X_test)\n",
354 "y_proba_list = list(zip(y_test[0:20], y_proba_lr[0:20,1]))\n",
355 "\n",
356 "# show the probability of positive class for first 20 instances\n",
357 "y_proba_list"
358 ]
359 },
360 {
361 "cell_type": "markdown",
362 "metadata": {},
363 "source": [
364 "### Precision-recall curves"
365 ]
366 },
367 {
368 "cell_type": "code",
369 "execution_count": null,
370 "metadata": {
371 "collapsed": false
372 },
373 "outputs": [],
374 "source": [
375 "from sklearn.metrics import precision_recall_curve\n",
376 "\n",
377 "precision, recall, thresholds = precision_recall_curve(y_test, y_scores_lr)\n",
378 "closest_zero = np.argmin(np.abs(thresholds))\n",
379 "closest_zero_p = precision[closest_zero]\n",
380 "closest_zero_r = recall[closest_zero]\n",
381 "\n",
382 "plt.figure()\n",
383 "plt.xlim([0.0, 1.01])\n",
384 "plt.ylim([0.0, 1.01])\n",
385 "plt.plot(precision, recall, label='Precision-Recall Curve')\n",
386 "plt.plot(closest_zero_p, closest_zero_r, 'o', markersize = 12, fillstyle = 'none', c='r', mew=3)\n",
387 "plt.xlabel('Precision', fontsize=16)\n",
388 "plt.ylabel('Recall', fontsize=16)\n",
389 "plt.axes().set_aspect('equal')\n",
390 "plt.show()"
391 ]
392 },
393 {
394 "cell_type": "markdown",
395 "metadata": {},
396 "source": [
397 "### ROC curves, Area-Under-Curve (AUC)"
398 ]
399 },
400 {
401 "cell_type": "code",
402 "execution_count": null,
403 "metadata": {
404 "collapsed": false
405 },
406 "outputs": [],
407 "source": [
408 "from sklearn.metrics import roc_curve, auc\n",
409 "\n",
410 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
411 "\n",
412 "y_score_lr = lr.fit(X_train, y_train).decision_function(X_test)\n",
413 "fpr_lr, tpr_lr, _ = roc_curve(y_test, y_score_lr)\n",
414 "roc_auc_lr = auc(fpr_lr, tpr_lr)\n",
415 "\n",
416 "plt.figure()\n",
417 "plt.xlim([-0.01, 1.00])\n",
418 "plt.ylim([-0.01, 1.01])\n",
419 "plt.plot(fpr_lr, tpr_lr, lw=3, label='LogRegr ROC curve (area = {:0.2f})'.format(roc_auc_lr))\n",
420 "plt.xlabel('False Positive Rate', fontsize=16)\n",
421 "plt.ylabel('True Positive Rate', fontsize=16)\n",
422 "plt.title('ROC curve (1-of-10 digits classifier)', fontsize=16)\n",
423 "plt.legend(loc='lower right', fontsize=13)\n",
424 "plt.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n",
425 "plt.axes().set_aspect('equal')\n",
426 "plt.show()"
427 ]
428 },
429 {
430 "cell_type": "code",
431 "execution_count": null,
432 "metadata": {
433 "collapsed": false,
434 "scrolled": false
435 },
436 "outputs": [],
437 "source": [
438 "from matplotlib import cm\n",
439 "\n",
440 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
441 "\n",
442 "plt.figure()\n",
443 "plt.xlim([-0.01, 1.00])\n",
444 "plt.ylim([-0.01, 1.01])\n",
445 "for g in [0.01, 0.1, 0.20, 1]:\n",
446 " svm = SVC(gamma=g).fit(X_train, y_train)\n",
447 " y_score_svm = svm.decision_function(X_test)\n",
448 " fpr_svm, tpr_svm, _ = roc_curve(y_test, y_score_svm)\n",
449 " roc_auc_svm = auc(fpr_svm, tpr_svm)\n",
450 " accuracy_svm = svm.score(X_test, y_test)\n",
451 " print(\"gamma = {:.2f} accuracy = {:.2f} AUC = {:.2f}\".format(g, accuracy_svm, \n",
452 " roc_auc_svm))\n",
453 " plt.plot(fpr_svm, tpr_svm, lw=3, alpha=0.7, \n",
454 " label='SVM (gamma = {:0.2f}, area = {:0.2f})'.format(g, roc_auc_svm))\n",
455 "\n",
456 "plt.xlabel('False Positive Rate', fontsize=16)\n",
457 "plt.ylabel('True Positive Rate (Recall)', fontsize=16)\n",
458 "plt.plot([0, 1], [0, 1], color='k', lw=0.5, linestyle='--')\n",
459 "plt.legend(loc=\"lower right\", fontsize=11)\n",
460 "plt.title('ROC curve: (1-of-10 digits classifier)', fontsize=16)\n",
461 "plt.axes().set_aspect('equal')\n",
462 "\n",
463 "plt.show()"
464 ]
465 },
466 {
467 "cell_type": "markdown",
468 "metadata": {},
469 "source": [
470 "### Evaluation measures for multi-class classification"
471 ]
472 },
473 {
474 "cell_type": "markdown",
475 "metadata": {},
476 "source": [
477 "#### Multi-class confusion matrix"
478 ]
479 },
480 {
481 "cell_type": "code",
482 "execution_count": null,
483 "metadata": {
484 "collapsed": false,
485 "scrolled": false
486 },
487 "outputs": [],
488 "source": [
489 "dataset = load_digits()\n",
490 "X, y = dataset.data, dataset.target\n",
491 "X_train_mc, X_test_mc, y_train_mc, y_test_mc = train_test_split(X, y, random_state=0)\n",
492 "\n",
493 "\n",
494 "svm = SVC(kernel = 'linear').fit(X_train_mc, y_train_mc)\n",
495 "svm_predicted_mc = svm.predict(X_test_mc)\n",
496 "confusion_mc = confusion_matrix(y_test_mc, svm_predicted_mc)\n",
497 "df_cm = pd.DataFrame(confusion_mc, \n",
498 " index = [i for i in range(0,10)], columns = [i for i in range(0,10)])\n",
499 "\n",
500 "plt.figure(figsize=(5.5,4))\n",
501 "sns.heatmap(df_cm, annot=True)\n",
502 "plt.title('SVM Linear Kernel \\nAccuracy:{0:.3f}'.format(accuracy_score(y_test_mc, \n",
503 " svm_predicted_mc)))\n",
504 "plt.ylabel('True label')\n",
505 "plt.xlabel('Predicted label')\n",
506 "\n",
507 "\n",
508 "svm = SVC(kernel = 'rbf').fit(X_train_mc, y_train_mc)\n",
509 "svm_predicted_mc = svm.predict(X_test_mc)\n",
510 "confusion_mc = confusion_matrix(y_test_mc, svm_predicted_mc)\n",
511 "df_cm = pd.DataFrame(confusion_mc, index = [i for i in range(0,10)],\n",
512 " columns = [i for i in range(0,10)])\n",
513 "\n",
514 "plt.figure(figsize = (5.5,4))\n",
515 "sns.heatmap(df_cm, annot=True)\n",
516 "plt.title('SVM RBF Kernel \\nAccuracy:{0:.3f}'.format(accuracy_score(y_test_mc, \n",
517 " svm_predicted_mc)))\n",
518 "plt.ylabel('True label')\n",
519 "plt.xlabel('Predicted label');"
520 ]
521 },
522 {
523 "cell_type": "markdown",
524 "metadata": {},
525 "source": [
526 "#### Multi-class classification report"
527 ]
528 },
529 {
530 "cell_type": "code",
531 "execution_count": null,
532 "metadata": {
533 "collapsed": false
534 },
535 "outputs": [],
536 "source": [
537 "print(classification_report(y_test_mc, svm_predicted_mc))"
538 ]
539 },
540 {
541 "cell_type": "markdown",
542 "metadata": {},
543 "source": [
544 "#### Micro- vs. macro-averaged metrics"
545 ]
546 },
547 {
548 "cell_type": "code",
549 "execution_count": null,
550 "metadata": {
551 "collapsed": false
552 },
553 "outputs": [],
554 "source": [
555 "print('Micro-averaged precision = {:.2f} (treat instances equally)'\n",
556 " .format(precision_score(y_test_mc, svm_predicted_mc, average = 'micro')))\n",
557 "print('Macro-averaged precision = {:.2f} (treat classes equally)'\n",
558 " .format(precision_score(y_test_mc, svm_predicted_mc, average = 'macro')))"
559 ]
560 },
561 {
562 "cell_type": "code",
563 "execution_count": null,
564 "metadata": {
565 "collapsed": false
566 },
567 "outputs": [],
568 "source": [
569 "print('Micro-averaged f1 = {:.2f} (treat instances equally)'\n",
570 " .format(f1_score(y_test_mc, svm_predicted_mc, average = 'micro')))\n",
571 "print('Macro-averaged f1 = {:.2f} (treat classes equally)'\n",
572 " .format(f1_score(y_test_mc, svm_predicted_mc, average = 'macro')))"
573 ]
574 },
575 {
576 "cell_type": "markdown",
577 "metadata": {},
578 "source": [
579 "### Regression evaluation metrics"
580 ]
581 },
582 {
583 "cell_type": "code",
584 "execution_count": null,
585 "metadata": {
586 "collapsed": false
587 },
588 "outputs": [],
589 "source": [
590 "%matplotlib notebook\n",
591 "import matplotlib.pyplot as plt\n",
592 "import numpy as np\n",
593 "from sklearn.model_selection import train_test_split\n",
594 "from sklearn import datasets\n",
595 "from sklearn.linear_model import LinearRegression\n",
596 "from sklearn.metrics import mean_squared_error, r2_score\n",
597 "from sklearn.dummy import DummyRegressor\n",
598 "\n",
599 "diabetes = datasets.load_diabetes()\n",
600 "\n",
601 "X = diabetes.data[:, None, 6]\n",
602 "y = diabetes.target\n",
603 "\n",
604 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
605 "\n",
606 "lm = LinearRegression().fit(X_train, y_train)\n",
607 "lm_dummy_mean = DummyRegressor(strategy = 'mean').fit(X_train, y_train)\n",
608 "\n",
609 "y_predict = lm.predict(X_test)\n",
610 "y_predict_dummy_mean = lm_dummy_mean.predict(X_test)\n",
611 "\n",
612 "print('Linear model, coefficients: ', lm.coef_)\n",
613 "print(\"Mean squared error (dummy): {:.2f}\".format(mean_squared_error(y_test, \n",
614 " y_predict_dummy_mean)))\n",
615 "print(\"Mean squared error (linear model): {:.2f}\".format(mean_squared_error(y_test, y_predict)))\n",
616 "print(\"r2_score (dummy): {:.2f}\".format(r2_score(y_test, y_predict_dummy_mean)))\n",
617 "print(\"r2_score (linear model): {:.2f}\".format(r2_score(y_test, y_predict)))\n",
618 "\n",
619 "# Plot outputs\n",
620 "plt.scatter(X_test, y_test, color='black')\n",
621 "plt.plot(X_test, y_predict, color='green', linewidth=2)\n",
622 "plt.plot(X_test, y_predict_dummy_mean, color='red', linestyle = 'dashed', \n",
623 " linewidth=2, label = 'dummy')\n",
624 "\n",
625 "plt.show()"
626 ]
627 },
628 {
629 "cell_type": "markdown",
630 "metadata": {},
631 "source": [
632 "### Model selection using evaluation metrics"
633 ]
634 },
635 {
636 "cell_type": "markdown",
637 "metadata": {},
638 "source": [
639 "#### Cross-validation example"
640 ]
641 },
642 {
643 "cell_type": "code",
644 "execution_count": null,
645 "metadata": {
646 "collapsed": false
647 },
648 "outputs": [],
649 "source": [
650 "from sklearn.model_selection import cross_val_score\n",
651 "from sklearn.svm import SVC\n",
652 "\n",
653 "dataset = load_digits()\n",
654 "# again, making this a binary problem with 'digit 1' as positive class \n",
655 "# and 'not 1' as negative class\n",
656 "X, y = dataset.data, dataset.target == 1\n",
657 "clf = SVC(kernel='linear', C=1)\n",
658 "\n",
659 "# accuracy is the default scoring metric\n",
660 "print('Cross-validation (accuracy)', cross_val_score(clf, X, y, cv=5))\n",
661 "# use AUC as scoring metric\n",
662 "print('Cross-validation (AUC)', cross_val_score(clf, X, y, cv=5, scoring = 'roc_auc'))\n",
663 "# use recall as scoring metric\n",
664 "print('Cross-validation (recall)', cross_val_score(clf, X, y, cv=5, scoring = 'recall'))"
665 ]
666 },
667 {
668 "cell_type": "markdown",
669 "metadata": {},
670 "source": [
671 "#### Grid search example"
672 ]
673 },
674 {
675 "cell_type": "code",
676 "execution_count": null,
677 "metadata": {
678 "collapsed": false
679 },
680 "outputs": [],
681 "source": [
682 "from sklearn.svm import SVC\n",
683 "from sklearn.model_selection import GridSearchCV\n",
684 "from sklearn.metrics import roc_auc_score\n",
685 "\n",
686 "dataset = load_digits()\n",
687 "X, y = dataset.data, dataset.target == 1\n",
688 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
689 "\n",
690 "clf = SVC(kernel='rbf')\n",
691 "grid_values = {'gamma': [0.001, 0.01, 0.05, 0.1, 1, 10, 100]}\n",
692 "\n",
693 "# default metric to optimize over grid parameters: accuracy\n",
694 "grid_clf_acc = GridSearchCV(clf, param_grid = grid_values)\n",
695 "grid_clf_acc.fit(X_train, y_train)\n",
696 "y_decision_fn_scores_acc = grid_clf_acc.decision_function(X_test) \n",
697 "\n",
698 "print('Grid best parameter (max. accuracy): ', grid_clf_acc.best_params_)\n",
699 "print('Grid best score (accuracy): ', grid_clf_acc.best_score_)\n",
700 "\n",
701 "# alternative metric to optimize over grid parameters: AUC\n",
702 "grid_clf_auc = GridSearchCV(clf, param_grid = grid_values, scoring = 'roc_auc')\n",
703 "grid_clf_auc.fit(X_train, y_train)\n",
704 "y_decision_fn_scores_auc = grid_clf_auc.decision_function(X_test) \n",
705 "\n",
706 "print('Test set AUC: ', roc_auc_score(y_test, y_decision_fn_scores_auc))\n",
707 "print('Grid best parameter (max. AUC): ', grid_clf_auc.best_params_)\n",
708 "print('Grid best score (AUC): ', grid_clf_auc.best_score_)\n"
709 ]
710 },
711 {
712 "cell_type": "markdown",
713 "metadata": {},
714 "source": [
715 "#### Evaluation metrics supported for model selection"
716 ]
717 },
718 {
719 "cell_type": "code",
720 "execution_count": null,
721 "metadata": {
722 "collapsed": false
723 },
724 "outputs": [],
725 "source": [
726 "from sklearn.metrics.scorer import SCORERS\n",
727 "\n",
728 "print(sorted(list(SCORERS.keys())))"
729 ]
730 },
731 {
732 "cell_type": "markdown",
733 "metadata": {},
734 "source": [
735 "### Two-feature classification example using the digits dataset"
736 ]
737 },
738 {
739 "cell_type": "markdown",
740 "metadata": {},
741 "source": [
742 "#### Optimizing a classifier using different evaluation metrics"
743 ]
744 },
745 {
746 "cell_type": "code",
747 "execution_count": null,
748 "metadata": {
749 "collapsed": false,
750 "scrolled": false
751 },
752 "outputs": [],
753 "source": [
754 "from sklearn.datasets import load_digits\n",
755 "from sklearn.model_selection import train_test_split\n",
756 "from adspy_shared_utilities import plot_class_regions_for_classifier_subplot\n",
757 "from sklearn.svm import SVC\n",
758 "from sklearn.model_selection import GridSearchCV\n",
759 "\n",
760 "\n",
761 "dataset = load_digits()\n",
762 "X, y = dataset.data, dataset.target == 1\n",
763 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
764 "\n",
765 "# Create a two-feature input vector matching the example plot above\n",
766 "# We jitter the points (add a small amount of random noise) in case there are areas\n",
767 "# in feature space where many instances have the same features.\n",
768 "jitter_delta = 0.25\n",
769 "X_twovar_train = X_train[:,[20,59]]+ np.random.rand(X_train.shape[0], 2) - jitter_delta\n",
770 "X_twovar_test = X_test[:,[20,59]] + np.random.rand(X_test.shape[0], 2) - jitter_delta\n",
771 "\n",
772 "clf = SVC(kernel = 'linear').fit(X_twovar_train, y_train)\n",
773 "grid_values = {'class_weight':['balanced', {1:2},{1:3},{1:4},{1:5},{1:10},{1:20},{1:50}]}\n",
774 "plt.figure(figsize=(9,6))\n",
775 "for i, eval_metric in enumerate(('precision','recall', 'f1','roc_auc')):\n",
776 " grid_clf_custom = GridSearchCV(clf, param_grid=grid_values, scoring=eval_metric)\n",
777 " grid_clf_custom.fit(X_twovar_train, y_train)\n",
778 " print('Grid best parameter (max. {0}): {1}'\n",
779 " .format(eval_metric, grid_clf_custom.best_params_))\n",
780 " print('Grid best score ({0}): {1}'\n",
781 " .format(eval_metric, grid_clf_custom.best_score_))\n",
782 " plt.subplots_adjust(wspace=0.3, hspace=0.3)\n",
783 " plot_class_regions_for_classifier_subplot(grid_clf_custom, X_twovar_test, y_test, None,\n",
784 " None, None, plt.subplot(2, 2, i+1))\n",
785 " \n",
786 " plt.title(eval_metric+'-oriented SVC')\n",
787 "plt.tight_layout()\n",
788 "plt.show()"
789 ]
790 },
791 {
792 "cell_type": "markdown",
793 "metadata": {},
794 "source": [
795 "#### Precision-recall curve for the default SVC classifier (with balanced class weights)"
796 ]
797 },
798 {
799 "cell_type": "code",
800 "execution_count": null,
801 "metadata": {
802 "collapsed": false,
803 "scrolled": false
804 },
805 "outputs": [],
806 "source": [
807 "from sklearn.model_selection import train_test_split\n",
808 "from sklearn.metrics import precision_recall_curve\n",
809 "from adspy_shared_utilities import plot_class_regions_for_classifier\n",
810 "from sklearn.svm import SVC\n",
811 "\n",
812 "dataset = load_digits()\n",
813 "X, y = dataset.data, dataset.target == 1\n",
814 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
815 "\n",
816 "# create a two-feature input vector matching the example plot above\n",
817 "jitter_delta = 0.25\n",
818 "X_twovar_train = X_train[:,[20,59]]+ np.random.rand(X_train.shape[0], 2) - jitter_delta\n",
819 "X_twovar_test = X_test[:,[20,59]] + np.random.rand(X_test.shape[0], 2) - jitter_delta\n",
820 "\n",
821 "clf = SVC(kernel='linear', class_weight='balanced').fit(X_twovar_train, y_train)\n",
822 "\n",
823 "y_scores = clf.decision_function(X_twovar_test)\n",
824 "\n",
825 "precision, recall, thresholds = precision_recall_curve(y_test, y_scores)\n",
826 "closest_zero = np.argmin(np.abs(thresholds))\n",
827 "closest_zero_p = precision[closest_zero]\n",
828 "closest_zero_r = recall[closest_zero]\n",
829 "\n",
830 "plot_class_regions_for_classifier(clf, X_twovar_test, y_test)\n",
831 "plt.title(\"SVC, class_weight = 'balanced', optimized for accuracy\")\n",
832 "plt.show()\n",
833 "\n",
834 "plt.figure()\n",
835 "plt.xlim([0.0, 1.01])\n",
836 "plt.ylim([0.0, 1.01])\n",
837 "plt.title (\"Precision-recall curve: SVC, class_weight = 'balanced'\")\n",
838 "plt.plot(precision, recall, label = 'Precision-Recall Curve')\n",
839 "plt.plot(closest_zero_p, closest_zero_r, 'o', markersize=12, fillstyle='none', c='r', mew=3)\n",
840 "plt.xlabel('Precision', fontsize=16)\n",
841 "plt.ylabel('Recall', fontsize=16)\n",
842 "plt.axes().set_aspect('equal')\n",
843 "plt.show()\n",
844 "print('At zero threshold, precision: {:.2f}, recall: {:.2f}'\n",
845 " .format(closest_zero_p, closest_zero_r))"
846 ]
847 },
848 {
849 "cell_type": "code",
850 "execution_count": null,
851 "metadata": {
852 "collapsed": true
853 },
854 "outputs": [],
855 "source": []
856 }
857 ],
858 "metadata": {
859 "anaconda-cloud": {},
860 "kernelspec": {
861 "display_name": "Python 3",
862 "language": "python",
863 "name": "python3"
864 },
865 "language_info": {
866 "codemirror_mode": {
867 "name": "ipython",
868 "version": 3
869 },
870 "file_extension": ".py",
871 "mimetype": "text/x-python",
872 "name": "python",
873 "nbconvert_exporter": "python",
874 "pygments_lexer": "ipython3",
875 "version": "3.6.2"
876 }
877 },
878 "nbformat": 4,
879 "nbformat_minor": 1
880 }
881