{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Chapter 4 - Classification"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"- [Load dataset](#Load-dataset)\n",
"- [The Default data set](#Figure-4.1---Default-data-set)\n",
"- [4.3 Logistic Regression](#4.3-Logistic-Regression)\n",
"- [4.4 Linear Discriminant Analysis](#4.4-Linear-Discriminant-Analysis)\n",
"- [Lab: 4.6.3 Linear Discriminant Analysis](#4.6.3-Linear-Discriminant-Analysis)\n",
"- [Lab: 4.6.4 Quadratic Discriminant Analysis](#4.6.4-Quadratic-Discriminant-Analysis)\n",
"- [Lab: 4.6.5 K-Nearest Neighbors](#4.6.5-K-Nearest-Neighbors)\n",
"- [Lab: 4.6.6 An Application to Caravan Insurance Data](#4.6.6-An-Application-to-Caravan-Insurance-Data)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib as mpl\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"import sklearn.linear_model as skl_lm\n",
"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
"from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
"from sklearn.metrics import confusion_matrix, classification_report, precision_score\n",
"from sklearn import preprocessing\n",
"from sklearn import neighbors\n",
"\n",
"import statsmodels.api as sm\n",
"import statsmodels.formula.api as smf\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### Load dataset"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/tz89p/Library/Python/3.9/lib/python/site-packages/openpyxl/styles/stylesheet.py:226: UserWarning: Workbook contains no default style, apply openpyxl's default\n",
" warn(\"Workbook contains no default style, apply openpyxl's default\")\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Unnamed: 0 | \n",
" default | \n",
" student | \n",
" balance | \n",
" income | \n",
" default2 | \n",
" student2 | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 1 | \n",
" No | \n",
" No | \n",
" 729.526495 | \n",
" 44361.625074 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" | 1 | \n",
" 2 | \n",
" No | \n",
" Yes | \n",
" 817.180407 | \n",
" 12106.134700 | \n",
" 0 | \n",
" 1 | \n",
"
\n",
" \n",
" | 2 | \n",
" 3 | \n",
" No | \n",
" No | \n",
" 1073.549164 | \n",
" 31767.138947 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"X_train = df.balance.values.reshape(-1,1) \n",
"y = df.default2\n",
"\n",
"# Create array of test data. Calculate the classification probability\n",
"# and predicted classification.\n",
"X_test = np.arange(df.balance.min(), df.balance.max()).reshape(-1,1)\n",
"\n",
"clf = skl_lm.LogisticRegression(solver='newton-cg')\n",
"clf.fit(X_train,y)\n",
"prob = clf.predict_proba(X_test) ## return the probability of the classes (n, n_classes)\n",
"\n",
"fig, (ax1, ax2) = plt.subplots(1,2, figsize=(12,5))\n",
"# Left plot\n",
"sns.regplot(data=df, x='balance', y='default2', order=1, ci=None,\n",
" scatter_kws={'color':'orange'},\n",
" line_kws={'color':'lightblue', 'lw':2}, ax=ax1)\n",
"# Right plot\n",
"ax2.scatter(X_train, y, color='orange')\n",
"ax2.plot(X_test, prob[:,1], color='lightblue')\n",
"\n",
"for ax in fig.axes:\n",
" ax.hlines(1, xmin=ax.xaxis.get_data_interval()[0],\n",
" xmax=ax.xaxis.get_data_interval()[1], linestyles='dashed', lw=1)\n",
" ax.hlines(0, xmin=ax.xaxis.get_data_interval()[0],\n",
" xmax=ax.xaxis.get_data_interval()[1], linestyles='dashed', lw=1)\n",
" ax.set_ylabel('Probability of default')\n",
" ax.set_xlabel('Balance')\n",
" ax.set_yticks([0, 0.25, 0.5, 0.75, 1.])\n",
" ax.set_xlim(xmin=-100)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### scikit-learn"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"LogisticRegression(solver='newton-cg')\n",
"classes: [0 1]\n",
"coefficients: [[0.00549891]]\n",
"intercept : [-10.6513227]\n"
]
}
],
"source": [
"# Using newton-cg solver, the coefficients are equal/closest to the ones in the book. \n",
"# I do not know the details on the differences between the solvers.\n",
"clf = skl_lm.LogisticRegression(solver='newton-cg')\n",
"X_train = df.balance.values.reshape(-1,1)\n",
"y = df.default2\n",
"clf.fit(X_train,y)\n",
"print(clf)\n",
"print('classes: ',clf.classes_)\n",
"print('coefficients: ',clf.coef_)\n",
"print('intercept :', clf.intercept_)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### statsmodels"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Optimization terminated successfully.\n",
" Current function value: 0.079823\n",
" Iterations 10\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Coef. | \n",
" Std.Err. | \n",
" z | \n",
" P>|z| | \n",
" [0.025 | \n",
" 0.975] | \n",
"
\n",
" \n",
" \n",
" \n",
" | Intercept | \n",
" -10.651331 | \n",
" 0.361169 | \n",
" -29.491287 | \n",
" 3.723665e-191 | \n",
" -11.359208 | \n",
" -9.943453 | \n",
"
\n",
" \n",
" | balance | \n",
" 0.005499 | \n",
" 0.000220 | \n",
" 24.952404 | \n",
" 2.010855e-137 | \n",
" 0.005067 | \n",
" 0.005931 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Coef. | \n",
" Std.Err. | \n",
" z | \n",
" P>|z| | \n",
" [0.025 | \n",
" 0.975] | \n",
"
\n",
" \n",
" \n",
" \n",
" | Intercept | \n",
" -3.504128 | \n",
" 0.070713 | \n",
" -49.554094 | \n",
" 0.000000 | \n",
" -3.642723 | \n",
" -3.365532 | \n",
"
\n",
" \n",
" | student2 | \n",
" 0.404887 | \n",
" 0.115019 | \n",
" 3.520177 | \n",
" 0.000431 | \n",
" 0.179454 | \n",
" 0.630320 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Coef. | \n",
" Std.Err. | \n",
" z | \n",
" P>|z| | \n",
" [0.025 | \n",
" 0.975] | \n",
"
\n",
" \n",
" \n",
" \n",
" | Intercept | \n",
" -10.869045 | \n",
" 0.492273 | \n",
" -22.079320 | \n",
" 4.995499e-108 | \n",
" -11.833882 | \n",
" -9.904209 | \n",
"
\n",
" \n",
" | balance | \n",
" 0.005737 | \n",
" 0.000232 | \n",
" 24.736506 | \n",
" 4.331521e-135 | \n",
" 0.005282 | \n",
" 0.006191 | \n",
"
\n",
" \n",
" | income | \n",
" 0.000003 | \n",
" 0.000008 | \n",
" 0.369808 | \n",
" 7.115254e-01 | \n",
" -0.000013 | \n",
" 0.000019 | \n",
"
\n",
" \n",
" | student2 | \n",
" -0.646776 | \n",
" 0.236257 | \n",
" -2.737595 | \n",
" 6.189022e-03 | \n",
" -1.109831 | \n",
" -0.183721 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | default | \n",
" No | \n",
" Yes | \n",
"
\n",
" \n",
" | student | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | No | \n",
" 6850 | \n",
" 206 | \n",
"
\n",
" \n",
" | Yes | \n",
" 2817 | \n",
" 127 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# creating plot\n",
"fig, (ax1, ax2) = plt.subplots(1,2, figsize=(12,5))\n",
"\n",
"# Left plot\n",
"ax1.plot(X_test, pd.DataFrame(prob)[1], color='orange', label='Student')\n",
"ax1.plot(X_test, pd.DataFrame(prob2)[1], color='lightblue', label='Non-student')\n",
"\n",
"ax1.set_ylabel('Default Rate')\n",
"ax1.set_xlabel('Credit Card Balance')\n",
"ax1.set_yticks([0, 0.2, 0.4, 0.6, 0.8, 1.])\n",
"ax1.set_xlim(450,2500)\n",
"ax1.legend(loc=2)\n",
"\n",
"# Right plot\n",
"sns.boxplot(x='student', y='balance', data=df, orient='v', ax=ax2, palette=c_palette)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### 4.4 Linear Discriminant Analysis\n"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | True default status | \n",
" No | \n",
" Yes | \n",
"
\n",
" \n",
" | Predicted default status | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | No | \n",
" 9645 | \n",
" 254 | \n",
"
\n",
" \n",
" | Yes | \n",
" 22 | \n",
" 79 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | True default status | \n",
" No | \n",
" Yes | \n",
"
\n",
" \n",
" | Predicted default status | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | No | \n",
" 9645 | \n",
" 254 | \n",
"
\n",
" \n",
" | Yes | \n",
" 22 | \n",
" 79 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | True default status | \n",
" No | \n",
" Yes | \n",
"
\n",
" \n",
" | Predicted default status | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | False | \n",
" 9518 | \n",
" 164 | \n",
"
\n",
" \n",
" | True | \n",
" 149 | \n",
" 169 | \n",
"
\n",
" \n",
"
\n",
"