Format: application/octet-stream
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "metadata": {},
6 "source": [
7 "---\n",
8 "\n",
9 "_You are currently looking at **version 1.0** of this notebook. To download notebooks and datafiles, as well as get help on Jupyter notebooks in the Coursera platform, visit the [Jupyter Notebook FAQ](https://www.coursera.org/learn/python-machine-learning/resources/bANLa) course resource._\n",
10 "\n",
11 "---"
12 ]
13 },
14 {
15 "cell_type": "markdown",
16 "metadata": {
17 "collapsed": true
18 },
19 "source": [
20 "# Applied Machine Learning: Module 3 (Evaluation)"
21 ]
22 },
23 {
24 "cell_type": "markdown",
25 "metadata": {},
26 "source": [
27 "## Evaluation for Classification"
28 ]
29 },
30 {
31 "cell_type": "markdown",
32 "metadata": {},
33 "source": [
34 "### Preamble"
35 ]
36 },
37 {
38 "cell_type": "code",
39 "execution_count": null,
40 "metadata": {},
41 "outputs": [],
42 "source": [
43 "%matplotlib notebook\n",
44 "import numpy as np\n",
45 "import pandas as pd\n",
46 "import seaborn as sns\n",
47 "import matplotlib.pyplot as plt\n",
48 "from sklearn.model_selection import train_test_split\n",
49 "from sklearn.datasets import load_digits\n",
50 "\n",
51 "dataset = load_digits()\n",
52 "X, y = dataset.data, dataset.target\n",
53 "\n",
54 "for class_name, class_count in zip(dataset.target_names, np.bincount(dataset.target)):\n",
55 " print(class_name,class_count)"
56 ]
57 },
58 {
59 "cell_type": "code",
60 "execution_count": null,
61 "metadata": {},
62 "outputs": [],
63 "source": [
64 "# Creating a dataset with imbalanced binary classes: \n",
65 "# Negative class (0) is 'not digit 1' \n",
66 "# Positive class (1) is 'digit 1'\n",
67 "y_binary_imbalanced = y.copy()\n",
68 "y_binary_imbalanced[y_binary_imbalanced != 1] = 0\n",
69 "\n",
70 "print('Original labels:\\t', y[1:30])\n",
71 "print('New binary labels:\\t', y_binary_imbalanced[1:30])"
72 ]
73 },
74 {
75 "cell_type": "code",
76 "execution_count": null,
77 "metadata": {
78 "scrolled": true
79 },
80 "outputs": [],
81 "source": [
82 "np.bincount(y_binary_imbalanced) # Negative class (0) is the most frequent class"
83 ]
84 },
85 {
86 "cell_type": "code",
87 "execution_count": null,
88 "metadata": {},
89 "outputs": [],
90 "source": [
91 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
92 "\n",
93 "# Accuracy of Support Vector Machine classifier\n",
94 "from sklearn.svm import SVC\n",
95 "\n",
96 "svm = SVC(kernel='rbf', C=1).fit(X_train, y_train)\n",
97 "svm.score(X_test, y_test)"
98 ]
99 },
100 {
101 "cell_type": "markdown",
102 "metadata": {},
103 "source": [
104 "### Dummy Classifiers"
105 ]
106 },
107 {
108 "cell_type": "markdown",
109 "metadata": {
110 "collapsed": true
111 },
112 "source": [
113 "DummyClassifier is a classifier that makes predictions using simple rules, which can be useful as a baseline for comparison against actual classifiers, especially with imbalanced classes."
114 ]
115 },
116 {
117 "cell_type": "code",
118 "execution_count": null,
119 "metadata": {},
120 "outputs": [],
121 "source": [
122 "from sklearn.dummy import DummyClassifier\n",
123 "\n",
124 "# Negative class (0) is most frequent\n",
125 "dummy_majority = DummyClassifier(strategy = 'most_frequent').fit(X_train, y_train)\n",
126 "# Therefore the dummy 'most_frequent' classifier always predicts class 0\n",
127 "y_dummy_predictions = dummy_majority.predict(X_test)\n",
128 "\n",
129 "y_dummy_predictions"
130 ]
131 },
132 {
133 "cell_type": "code",
134 "execution_count": null,
135 "metadata": {},
136 "outputs": [],
137 "source": [
138 "dummy_majority.score(X_test, y_test)"
139 ]
140 },
141 {
142 "cell_type": "code",
143 "execution_count": null,
144 "metadata": {},
145 "outputs": [],
146 "source": [
147 "svm = SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
148 "svm.score(X_test, y_test)"
149 ]
150 },
151 {
152 "cell_type": "markdown",
153 "metadata": {},
154 "source": [
155 "### Confusion matrices"
156 ]
157 },
158 {
159 "cell_type": "markdown",
160 "metadata": {},
161 "source": [
162 "#### Binary (two-class) confusion matrix"
163 ]
164 },
165 {
166 "cell_type": "code",
167 "execution_count": null,
168 "metadata": {},
169 "outputs": [],
170 "source": [
171 "from sklearn.metrics import confusion_matrix\n",
172 "\n",
173 "# Negative class (0) is most frequent\n",
174 "dummy_majority = DummyClassifier(strategy = 'most_frequent').fit(X_train, y_train)\n",
175 "y_majority_predicted = dummy_majority.predict(X_test)\n",
176 "confusion = confusion_matrix(y_test, y_majority_predicted)\n",
177 "\n",
178 "print('Most frequent class (dummy classifier)\\n', confusion)"
179 ]
180 },
181 {
182 "cell_type": "code",
183 "execution_count": null,
184 "metadata": {},
185 "outputs": [],
186 "source": [
187 "# produces random predictions w/ same class proportion as training set\n",
188 "dummy_classprop = DummyClassifier(strategy='stratified').fit(X_train, y_train)\n",
189 "y_classprop_predicted = dummy_classprop.predict(X_test)\n",
190 "confusion = confusion_matrix(y_test, y_classprop_predicted)\n",
191 "\n",
192 "print('Random class-proportional prediction (dummy classifier)\\n', confusion)"
193 ]
194 },
195 {
196 "cell_type": "code",
197 "execution_count": null,
198 "metadata": {
199 "scrolled": true
200 },
201 "outputs": [],
202 "source": [
203 "svm = SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
204 "svm_predicted = svm.predict(X_test)\n",
205 "confusion = confusion_matrix(y_test, svm_predicted)\n",
206 "\n",
207 "print('Support vector machine classifier (linear kernel, C=1)\\n', confusion)"
208 ]
209 },
210 {
211 "cell_type": "code",
212 "execution_count": null,
213 "metadata": {},
214 "outputs": [],
215 "source": [
216 "from sklearn.linear_model import LogisticRegression\n",
217 "\n",
218 "lr = LogisticRegression().fit(X_train, y_train)\n",
219 "lr_predicted = lr.predict(X_test)\n",
220 "confusion = confusion_matrix(y_test, lr_predicted)\n",
221 "\n",
222 "print('Logistic regression classifier (default settings)\\n', confusion)"
223 ]
224 },
225 {
226 "cell_type": "code",
227 "execution_count": null,
228 "metadata": {},
229 "outputs": [],
230 "source": [
231 "from sklearn.tree import DecisionTreeClassifier\n",
232 "\n",
233 "dt = DecisionTreeClassifier(max_depth=2).fit(X_train, y_train)\n",
234 "tree_predicted = dt.predict(X_test)\n",
235 "confusion = confusion_matrix(y_test, tree_predicted)\n",
236 "\n",
237 "print('Decision tree classifier (max_depth = 2)\\n', confusion)"
238 ]
239 },
240 {
241 "cell_type": "markdown",
242 "metadata": {},
243 "source": [
244 "### Evaluation metrics for binary classification"
245 ]
246 },
247 {
248 "cell_type": "code",
249 "execution_count": null,
250 "metadata": {},
251 "outputs": [],
252 "source": [
253 "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
254 "# Accuracy = TP + TN / (TP + TN + FP + FN)\n",
255 "# Precision = TP / (TP + FP)\n",
256 "# Recall = TP / (TP + FN) Also known as sensitivity, or True Positive Rate\n",
257 "# F1 = 2 * Precision * Recall / (Precision + Recall) \n",
258 "print('Accuracy: {:.2f}'.format(accuracy_score(y_test, tree_predicted)))\n",
259 "print('Precision: {:.2f}'.format(precision_score(y_test, tree_predicted)))\n",
260 "print('Recall: {:.2f}'.format(recall_score(y_test, tree_predicted)))\n",
261 "print('F1: {:.2f}'.format(f1_score(y_test, tree_predicted)))"
262 ]
263 },
264 {
265 "cell_type": "code",
266 "execution_count": null,
267 "metadata": {},
268 "outputs": [],
269 "source": [
270 "# Combined report with all above metrics\n",
271 "from sklearn.metrics import classification_report\n",
272 "\n",
273 "print(classification_report(y_test, tree_predicted, target_names=['not 1', '1']))"
274 ]
275 },
276 {
277 "cell_type": "code",
278 "execution_count": null,
279 "metadata": {
280 "scrolled": false
281 },
282 "outputs": [],
283 "source": [
284 "print('Random class-proportional (dummy)\\n', \n",
285 " classification_report(y_test, y_classprop_predicted, target_names=['not 1', '1']))\n",
286 "print('SVM\\n', \n",
287 " classification_report(y_test, svm_predicted, target_names = ['not 1', '1']))\n",
288 "print('Logistic regression\\n', \n",
289 " classification_report(y_test, lr_predicted, target_names = ['not 1', '1']))\n",
290 "print('Decision tree\\n', \n",
291 " classification_report(y_test, tree_predicted, target_names = ['not 1', '1']))"
292 ]
293 },
294 {
295 "cell_type": "markdown",
296 "metadata": {},
297 "source": [
298 "### Decision functions"
299 ]
300 },
301 {
302 "cell_type": "code",
303 "execution_count": null,
304 "metadata": {},
305 "outputs": [],
306 "source": [
307 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
308 "y_scores_lr = lr.fit(X_train, y_train).decision_function(X_test)\n",
309 "y_score_list = list(zip(y_test[0:20], y_scores_lr[0:20]))\n",
310 "\n",
311 "# show the decision_function scores for first 20 instances\n",
312 "y_score_list"
313 ]
314 },
315 {
316 "cell_type": "code",
317 "execution_count": null,
318 "metadata": {},
319 "outputs": [],
320 "source": [
321 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
322 "y_proba_lr = lr.fit(X_train, y_train).predict_proba(X_test)\n",
323 "y_proba_list = list(zip(y_test[0:20], y_proba_lr[0:20,1]))\n",
324 "\n",
325 "# show the probability of positive class for first 20 instances\n",
326 "y_proba_list"
327 ]
328 },
329 {
330 "cell_type": "markdown",
331 "metadata": {},
332 "source": [
333 "### Precision-recall curves"
334 ]
335 },
336 {
337 "cell_type": "code",
338 "execution_count": null,
339 "metadata": {},
340 "outputs": [],
341 "source": [
342 "from sklearn.metrics import precision_recall_curve\n",
343 "\n",
344 "precision, recall, thresholds = precision_recall_curve(y_test, y_scores_lr)\n",
345 "closest_zero = np.argmin(np.abs(thresholds))\n",
346 "closest_zero_p = precision[closest_zero]\n",
347 "closest_zero_r = recall[closest_zero]\n",
348 "\n",
349 "plt.figure()\n",
350 "plt.xlim([0.0, 1.01])\n",
351 "plt.ylim([0.0, 1.01])\n",
352 "plt.plot(precision, recall, label='Precision-Recall Curve')\n",
353 "plt.plot(closest_zero_p, closest_zero_r, 'o', markersize = 12, fillstyle = 'none', c='r', mew=3)\n",
354 "plt.xlabel('Precision', fontsize=16)\n",
355 "plt.ylabel('Recall', fontsize=16)\n",
356 "plt.axes().set_aspect('equal')\n",
357 "plt.show()"
358 ]
359 },
360 {
361 "cell_type": "markdown",
362 "metadata": {},
363 "source": [
364 "### ROC curves, Area-Under-Curve (AUC)"
365 ]
366 },
367 {
368 "cell_type": "code",
369 "execution_count": null,
370 "metadata": {},
371 "outputs": [],
372 "source": [
373 "from sklearn.metrics import roc_curve, auc\n",
374 "\n",
375 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
376 "\n",
377 "y_score_lr = lr.fit(X_train, y_train).decision_function(X_test)\n",
378 "fpr_lr, tpr_lr, _ = roc_curve(y_test, y_score_lr)\n",
379 "roc_auc_lr = auc(fpr_lr, tpr_lr)\n",
380 "\n",
381 "plt.figure()\n",
382 "plt.xlim([-0.01, 1.00])\n",
383 "plt.ylim([-0.01, 1.01])\n",
384 "plt.plot(fpr_lr, tpr_lr, lw=3, label='LogRegr ROC curve (area = {:0.2f})'.format(roc_auc_lr))\n",
385 "plt.xlabel('False Positive Rate', fontsize=16)\n",
386 "plt.ylabel('True Positive Rate', fontsize=16)\n",
387 "plt.title('ROC curve (1-of-10 digits classifier)', fontsize=16)\n",
388 "plt.legend(loc='lower right', fontsize=13)\n",
389 "plt.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n",
390 "plt.axes().set_aspect('equal')\n",
391 "plt.show()"
392 ]
393 },
394 {
395 "cell_type": "code",
396 "execution_count": null,
397 "metadata": {
398 "scrolled": false
399 },
400 "outputs": [],
401 "source": [
402 "from matplotlib import cm\n",
403 "\n",
404 "X_train, X_test, y_train, y_test = train_test_split(X, y_binary_imbalanced, random_state=0)\n",
405 "\n",
406 "plt.figure()\n",
407 "plt.xlim([-0.01, 1.00])\n",
408 "plt.ylim([-0.01, 1.01])\n",
409 "for g in [0.01, 0.1, 0.20, 1]:\n",
410 " svm = SVC(gamma=g).fit(X_train, y_train)\n",
411 " y_score_svm = svm.decision_function(X_test)\n",
412 " fpr_svm, tpr_svm, _ = roc_curve(y_test, y_score_svm)\n",
413 " roc_auc_svm = auc(fpr_svm, tpr_svm)\n",
414 " accuracy_svm = svm.score(X_test, y_test)\n",
415 " print(\"gamma = {:.2f} accuracy = {:.2f} AUC = {:.2f}\".format(g, accuracy_svm, \n",
416 " roc_auc_svm))\n",
417 " plt.plot(fpr_svm, tpr_svm, lw=3, alpha=0.7, \n",
418 " label='SVM (gamma = {:0.2f}, area = {:0.2f})'.format(g, roc_auc_svm))\n",
419 "\n",
420 "plt.xlabel('False Positive Rate', fontsize=16)\n",
421 "plt.ylabel('True Positive Rate (Recall)', fontsize=16)\n",
422 "plt.plot([0, 1], [0, 1], color='k', lw=0.5, linestyle='--')\n",
423 "plt.legend(loc=\"lower right\", fontsize=11)\n",
424 "plt.title('ROC curve: (1-of-10 digits classifier)', fontsize=16)\n",
425 "plt.axes().set_aspect('equal')\n",
426 "\n",
427 "plt.show()"
428 ]
429 },
430 {
431 "cell_type": "markdown",
432 "metadata": {},
433 "source": [
434 "### Evaluation measures for multi-class classification"
435 ]
436 },
437 {
438 "cell_type": "markdown",
439 "metadata": {},
440 "source": [
441 "#### Multi-class confusion matrix"
442 ]
443 },
444 {
445 "cell_type": "code",
446 "execution_count": null,
447 "metadata": {
448 "scrolled": false
449 },
450 "outputs": [],
451 "source": [
452 "dataset = load_digits()\n",
453 "X, y = dataset.data, dataset.target\n",
454 "X_train_mc, X_test_mc, y_train_mc, y_test_mc = train_test_split(X, y, random_state=0)\n",
455 "\n",
456 "\n",
457 "svm = SVC(kernel = 'linear').fit(X_train_mc, y_train_mc)\n",
458 "svm_predicted_mc = svm.predict(X_test_mc)\n",
459 "confusion_mc = confusion_matrix(y_test_mc, svm_predicted_mc)\n",
460 "df_cm = pd.DataFrame(confusion_mc, \n",
461 " index = [i for i in range(0,10)], columns = [i for i in range(0,10)])\n",
462 "\n",
463 "plt.figure(figsize=(5.5,4))\n",
464 "sns.heatmap(df_cm, annot=True)\n",
465 "plt.title('SVM Linear Kernel \\nAccuracy:{0:.3f}'.format(accuracy_score(y_test_mc, \n",
466 " svm_predicted_mc)))\n",
467 "plt.ylabel('True label')\n",
468 "plt.xlabel('Predicted label')\n",
469 "\n",
470 "\n",
471 "svm = SVC(kernel = 'rbf').fit(X_train_mc, y_train_mc)\n",
472 "svm_predicted_mc = svm.predict(X_test_mc)\n",
473 "confusion_mc = confusion_matrix(y_test_mc, svm_predicted_mc)\n",
474 "df_cm = pd.DataFrame(confusion_mc, index = [i for i in range(0,10)],\n",
475 " columns = [i for i in range(0,10)])\n",
476 "\n",
477 "plt.figure(figsize = (5.5,4))\n",
478 "sns.heatmap(df_cm, annot=True)\n",
479 "plt.title('SVM RBF Kernel \\nAccuracy:{0:.3f}'.format(accuracy_score(y_test_mc, \n",
480 " svm_predicted_mc)))\n",
481 "plt.ylabel('True label')\n",
482 "plt.xlabel('Predicted label');"
483 ]
484 },
485 {
486 "cell_type": "markdown",
487 "metadata": {},
488 "source": [
489 "#### Multi-class classification report"
490 ]
491 },
492 {
493 "cell_type": "code",
494 "execution_count": null,
495 "metadata": {},
496 "outputs": [],
497 "source": [
498 "print(classification_report(y_test_mc, svm_predicted_mc))"
499 ]
500 },
501 {
502 "cell_type": "markdown",
503 "metadata": {},
504 "source": [
505 "#### Micro- vs. macro-averaged metrics"
506 ]
507 },
508 {
509 "cell_type": "code",
510 "execution_count": null,
511 "metadata": {},
512 "outputs": [],
513 "source": [
514 "print('Micro-averaged precision = {:.2f} (treat instances equally)'\n",
515 " .format(precision_score(y_test_mc, svm_predicted_mc, average = 'micro')))\n",
516 "print('Macro-averaged precision = {:.2f} (treat classes equally)'\n",
517 " .format(precision_score(y_test_mc, svm_predicted_mc, average = 'macro')))"
518 ]
519 },
520 {
521 "cell_type": "code",
522 "execution_count": null,
523 "metadata": {},
524 "outputs": [],
525 "source": [
526 "print('Micro-averaged f1 = {:.2f} (treat instances equally)'\n",
527 " .format(f1_score(y_test_mc, svm_predicted_mc, average = 'micro')))\n",
528 "print('Macro-averaged f1 = {:.2f} (treat classes equally)'\n",
529 " .format(f1_score(y_test_mc, svm_predicted_mc, average = 'macro')))"
530 ]
531 },
532 {
533 "cell_type": "markdown",
534 "metadata": {},
535 "source": [
536 "### Regression evaluation metrics"
537 ]
538 },
539 {
540 "cell_type": "code",
541 "execution_count": null,
542 "metadata": {},
543 "outputs": [],
544 "source": [
545 "%matplotlib notebook\n",
546 "import matplotlib.pyplot as plt\n",
547 "import numpy as np\n",
548 "from sklearn.model_selection import train_test_split\n",
549 "from sklearn import datasets\n",
550 "from sklearn.linear_model import LinearRegression\n",
551 "from sklearn.metrics import mean_squared_error, r2_score\n",
552 "from sklearn.dummy import DummyRegressor\n",
553 "\n",
554 "diabetes = datasets.load_diabetes()\n",
555 "\n",
556 "X = diabetes.data[:, None, 6]\n",
557 "y = diabetes.target\n",
558 "\n",
559 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
560 "\n",
561 "lm = LinearRegression().fit(X_train, y_train)\n",
562 "lm_dummy_mean = DummyRegressor(strategy = 'mean').fit(X_train, y_train)\n",
563 "\n",
564 "y_predict = lm.predict(X_test)\n",
565 "y_predict_dummy_mean = lm_dummy_mean.predict(X_test)\n",
566 "\n",
567 "print('Linear model, coefficients: ', lm.coef_)\n",
568 "print(\"Mean squared error (dummy): {:.2f}\".format(mean_squared_error(y_test, \n",
569 " y_predict_dummy_mean)))\n",
570 "print(\"Mean squared error (linear model): {:.2f}\".format(mean_squared_error(y_test, y_predict)))\n",
571 "print(\"r2_score (dummy): {:.2f}\".format(r2_score(y_test, y_predict_dummy_mean)))\n",
572 "print(\"r2_score (linear model): {:.2f}\".format(r2_score(y_test, y_predict)))\n",
573 "\n",
574 "# Plot outputs\n",
575 "plt.scatter(X_test, y_test, color='black')\n",
576 "plt.plot(X_test, y_predict, color='green', linewidth=2)\n",
577 "plt.plot(X_test, y_predict_dummy_mean, color='red', linestyle = 'dashed', \n",
578 " linewidth=2, label = 'dummy')\n",
579 "\n",
580 "plt.show()"
581 ]
582 },
583 {
584 "cell_type": "markdown",
585 "metadata": {},
586 "source": [
587 "### Model selection using evaluation metrics"
588 ]
589 },
590 {
591 "cell_type": "markdown",
592 "metadata": {},
593 "source": [
594 "#### Cross-validation example"
595 ]
596 },
597 {
598 "cell_type": "code",
599 "execution_count": null,
600 "metadata": {},
601 "outputs": [],
602 "source": [
603 "from sklearn.model_selection import cross_val_score\n",
604 "from sklearn.svm import SVC\n",
605 "\n",
606 "dataset = load_digits()\n",
607 "# again, making this a binary problem with 'digit 1' as positive class \n",
608 "# and 'not 1' as negative class\n",
609 "X, y = dataset.data, dataset.target == 1\n",
610 "clf = SVC(kernel='linear', C=1)\n",
611 "\n",
612 "# accuracy is the default scoring metric\n",
613 "print('Cross-validation (accuracy)', cross_val_score(clf, X, y, cv=5))\n",
614 "# use AUC as scoring metric\n",
615 "print('Cross-validation (AUC)', cross_val_score(clf, X, y, cv=5, scoring = 'roc_auc'))\n",
616 "# use recall as scoring metric\n",
617 "print('Cross-validation (recall)', cross_val_score(clf, X, y, cv=5, scoring = 'recall'))"
618 ]
619 },
620 {
621 "cell_type": "markdown",
622 "metadata": {},
623 "source": [
624 "#### Grid search example"
625 ]
626 },
627 {
628 "cell_type": "code",
629 "execution_count": null,
630 "metadata": {},
631 "outputs": [],
632 "source": [
633 "from sklearn.svm import SVC\n",
634 "from sklearn.model_selection import GridSearchCV\n",
635 "from sklearn.metrics import roc_auc_score\n",
636 "\n",
637 "dataset = load_digits()\n",
638 "X, y = dataset.data, dataset.target == 1\n",
639 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
640 "\n",
641 "clf = SVC(kernel='rbf')\n",
642 "grid_values = {'gamma': [0.001, 0.01, 0.05, 0.1, 1, 10, 100]}\n",
643 "\n",
644 "# default metric to optimize over grid parameters: accuracy\n",
645 "grid_clf_acc = GridSearchCV(clf, param_grid = grid_values)\n",
646 "grid_clf_acc.fit(X_train, y_train)\n",
647 "y_decision_fn_scores_acc = grid_clf_acc.decision_function(X_test) \n",
648 "\n",
649 "print('Grid best parameter (max. accuracy): ', grid_clf_acc.best_params_)\n",
650 "print('Grid best score (accuracy): ', grid_clf_acc.best_score_)\n",
651 "\n",
652 "# alternative metric to optimize over grid parameters: AUC\n",
653 "grid_clf_auc = GridSearchCV(clf, param_grid = grid_values, scoring = 'roc_auc')\n",
654 "grid_clf_auc.fit(X_train, y_train)\n",
655 "y_decision_fn_scores_auc = grid_clf_auc.decision_function(X_test) \n",
656 "\n",
657 "print('Test set AUC: ', roc_auc_score(y_test, y_decision_fn_scores_auc))\n",
658 "print('Grid best parameter (max. AUC): ', grid_clf_auc.best_params_)\n",
659 "print('Grid best score (AUC): ', grid_clf_auc.best_score_)\n"
660 ]
661 },
662 {
663 "cell_type": "markdown",
664 "metadata": {},
665 "source": [
666 "#### Evaluation metrics supported for model selection"
667 ]
668 },
669 {
670 "cell_type": "code",
671 "execution_count": null,
672 "metadata": {},
673 "outputs": [],
674 "source": [
675 "from sklearn.metrics.scorer import SCORERS\n",
676 "\n",
677 "print(sorted(list(SCORERS.keys())))"
678 ]
679 },
680 {
681 "cell_type": "markdown",
682 "metadata": {},
683 "source": [
684 "### Two-feature classification example using the digits dataset"
685 ]
686 },
687 {
688 "cell_type": "markdown",
689 "metadata": {},
690 "source": [
691 "#### Optimizing a classifier using different evaluation metrics"
692 ]
693 },
694 {
695 "cell_type": "code",
696 "execution_count": null,
697 "metadata": {
698 "scrolled": false
699 },
700 "outputs": [],
701 "source": [
702 "from sklearn.datasets import load_digits\n",
703 "from sklearn.model_selection import train_test_split\n",
704 "from adspy_shared_utilities import plot_class_regions_for_classifier_subplot\n",
705 "from sklearn.svm import SVC\n",
706 "from sklearn.model_selection import GridSearchCV\n",
707 "\n",
708 "\n",
709 "dataset = load_digits()\n",
710 "X, y = dataset.data, dataset.target == 1\n",
711 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
712 "\n",
713 "# Create a two-feature input vector matching the example plot above\n",
714 "# We jitter the points (add a small amount of random noise) in case there are areas\n",
715 "# in feature space where many instances have the same features.\n",
716 "jitter_delta = 0.25\n",
717 "X_twovar_train = X_train[:,[20,59]]+ np.random.rand(X_train.shape[0], 2) - jitter_delta\n",
718 "X_twovar_test = X_test[:,[20,59]] + np.random.rand(X_test.shape[0], 2) - jitter_delta\n",
719 "\n",
720 "clf = SVC(kernel = 'linear').fit(X_twovar_train, y_train)\n",
721 "grid_values = {'class_weight':['balanced', {1:2},{1:3},{1:4},{1:5},{1:10},{1:20},{1:50}]}\n",
722 "plt.figure(figsize=(9,6))\n",
723 "for i, eval_metric in enumerate(('precision','recall', 'f1','roc_auc')):\n",
724 " grid_clf_custom = GridSearchCV(clf, param_grid=grid_values, scoring=eval_metric)\n",
725 " grid_clf_custom.fit(X_twovar_train, y_train)\n",
726 " print('Grid best parameter (max. {0}): {1}'\n",
727 " .format(eval_metric, grid_clf_custom.best_params_))\n",
728 " print('Grid best score ({0}): {1}'\n",
729 " .format(eval_metric, grid_clf_custom.best_score_))\n",
730 " plt.subplots_adjust(wspace=0.3, hspace=0.3)\n",
731 " plot_class_regions_for_classifier_subplot(grid_clf_custom, X_twovar_test, y_test, None,\n",
732 " None, None, plt.subplot(2, 2, i+1))\n",
733 " \n",
734 " plt.title(eval_metric+'-oriented SVC')\n",
735 "plt.tight_layout()\n",
736 "plt.show()"
737 ]
738 },
739 {
740 "cell_type": "markdown",
741 "metadata": {},
742 "source": [
743 "#### Precision-recall curve for the default SVC classifier (with balanced class weights)"
744 ]
745 },
746 {
747 "cell_type": "code",
748 "execution_count": null,
749 "metadata": {
750 "scrolled": false
751 },
752 "outputs": [],
753 "source": [
754 "from sklearn.model_selection import train_test_split\n",
755 "from sklearn.metrics import precision_recall_curve\n",
756 "from adspy_shared_utilities import plot_class_regions_for_classifier\n",
757 "from sklearn.svm import SVC\n",
758 "\n",
759 "dataset = load_digits()\n",
760 "X, y = dataset.data, dataset.target == 1\n",
761 "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
762 "\n",
763 "# create a two-feature input vector matching the example plot above\n",
764 "jitter_delta = 0.25\n",
765 "X_twovar_train = X_train[:,[20,59]]+ np.random.rand(X_train.shape[0], 2) - jitter_delta\n",
766 "X_twovar_test = X_test[:,[20,59]] + np.random.rand(X_test.shape[0], 2) - jitter_delta\n",
767 "\n",
768 "clf = SVC(kernel='linear', class_weight='balanced').fit(X_twovar_train, y_train)\n",
769 "\n",
770 "y_scores = clf.decision_function(X_twovar_test)\n",
771 "\n",
772 "precision, recall, thresholds = precision_recall_curve(y_test, y_scores)\n",
773 "closest_zero = np.argmin(np.abs(thresholds))\n",
774 "closest_zero_p = precision[closest_zero]\n",
775 "closest_zero_r = recall[closest_zero]\n",
776 "\n",
777 "plot_class_regions_for_classifier(clf, X_twovar_test, y_test)\n",
778 "plt.title(\"SVC, class_weight = 'balanced', optimized for accuracy\")\n",
779 "plt.show()\n",
780 "\n",
781 "plt.figure()\n",
782 "plt.xlim([0.0, 1.01])\n",
783 "plt.ylim([0.0, 1.01])\n",
784 "plt.title (\"Precision-recall curve: SVC, class_weight = 'balanced'\")\n",
785 "plt.plot(precision, recall, label = 'Precision-Recall Curve')\n",
786 "plt.plot(closest_zero_p, closest_zero_r, 'o', markersize=12, fillstyle='none', c='r', mew=3)\n",
787 "plt.xlabel('Precision', fontsize=16)\n",
788 "plt.ylabel('Recall', fontsize=16)\n",
789 "plt.axes().set_aspect('equal')\n",
790 "plt.show()\n",
791 "print('At zero threshold, precision: {:.2f}, recall: {:.2f}'\n",
792 " .format(closest_zero_p, closest_zero_r))"
793 ]
794 },
795 {
796 "cell_type": "code",
797 "execution_count": null,
798 "metadata": {
799 "collapsed": true
800 },
801 "outputs": [],
802 "source": []
803 }
804 ],
805 "metadata": {
806 "anaconda-cloud": {},
807 "kernelspec": {
808 "display_name": "Python 3",
809 "language": "python",
810 "name": "python3"
811 },
812 "language_info": {
813 "codemirror_mode": {
814 "name": "ipython",
815 "version": 3
816 },
817 "file_extension": ".py",
818 "mimetype": "text/x-python",
819 "name": "python",
820 "nbconvert_exporter": "python",
821 "pygments_lexer": "ipython3",
822 "version": "3.8.8"
823 }
824 },
825 "nbformat": 4,
826 "nbformat_minor": 1
827 }
828