{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.5.5\n",
      "IPython 6.2.1\n",
      "\n",
      "numpy 1.14.1\n",
      "scipy 1.0.0\n",
      "sklearn 0.19.1\n",
      "pandas 0.22.0\n",
      "matplotlib 2.2.2\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,scipy,sklearn,pandas,matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3장 – 분류**\n",
    "\n",
    "_이 노트북은 3장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "np.random.seed(42)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "matplotlib.rc('font', family='NanumBarunGothic')\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴드\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"classification\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'COL_NAMES': ['label', 'data'],\n",
       " 'DESCR': 'mldata.org dataset: mnist-original',\n",
       " 'data': array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),\n",
       " 'target': array([0., 0., 0., ..., 9., 9., 9.])}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "mnist = fetch_mldata('MNIST original')\n",
    "mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = mnist[\"data\"], mnist[\"target\"]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "784"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "28*28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARoAAAEYCAYAAACDezmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAABvtJREFUeJzt3T9rFF8bx+FZCRaSImiqICQIxsZCfBtB7NRG7awUIVpY2aQRRDtbQbHSQkS0TCEWYhe0CuJ/DAgRJE0KNb8XEJx7nmf3u1mz19WemzMHDB+nOMz2tra2GoCkPTt9AGD3ExogTmiAOKEB4oQGiBMaIE5ogDihAeKEBoib2IFnuooMu0evy5A3GiBOaIA4oQHihAaIExogTmiAOKEB4oQGiBMaIE5ogDihAeKEBogTGiBOaIA4oQHihAaIExogTmiAOKEB4oQGiBMaIE5ogDihAeKEBogTGiBOaIA4oQHihAaIExogTmiAOKEB4oQGiBMaIE5ogDihAeKEBogTGiBOaIA4oQHihAaIExogTmiAOKEB4iZ2+gAwCh4+fNi6/ubNm3KP+/fvD+o4rT59+jSU5wySNxogTmiAOKEB4oQGiBMaIE5ogDihAeLco2FkbWxslDMvX74sZ5aWlsqZV69eta73er1yD/7OGw0QJzRAnNAAcUIDxAkNECc0QJzQAHFCA8S5sMc2v379al1fW1sbyHOqi3QfPnwo91heXh7IWYZlenq6df3MmTNDOslweaMB4oQGiBMaIE5ogDihAeKEBogTGiBOaIA4F/bYprqQNzc3V+6xtbVVzvxLX607duxYOXP27NlyZmFhoXX98OHDnc/0L/FGA8QJDRAnNECc0ABxQgPECQ0QJzRAnHs0bHP16tXW9S53ZLrMVGZmZsqZCxculDPXr1/v+yz0xxsNECc0QJzQAHFCA8QJDRAnNECc0ABxQgPEubA3Zu7evVvOPH/+vHV9UB+sqvZZX18v96h+VbNpmmZ1dbWcmZ+fL2f4/3mjAeKEBogTGiBOaIA4oQHihAaIExogTmiAuN4gvoT2Pxr6A8dFl8t4i4uL5czGxkbfZxmlX6qcnZ0tZ96/fz+Ek+xKnf4RvdEAcUIDxAkNECc0QJzQAHFCA8QJDRAnNECcC3u7SJeLaV+/fu37OVNTU+XM5ORkObNnT/v/c5ubm+Ue379/L2e6+P3790D2GUMu7AGjQWiAOKEB4oQGiBMaIE5ogDihAeL8UuUucvLkyXLmzp075cz58+db1y9evFjucfz48XKmsra2Vs4sLCyUMysrK32fhf54owHihAaIExogTmiAOKEB4oQGiBMaIE5ogDgfvmJkffv2rZwZ1IW9P3/+dDoT2/jwFTAahAaIExogTmiAOKEB4oQGiBMaIE5ogDhf2Ct8+fKldX3fvn3lHgcOHBjUccZKl4t2vV59X6zLzJMnT1rXu3y9kL/zRgPECQ0QJzRAnNAAcUIDxAkNECc0QNxY36O5ceNGOXPv3r3W9b1795Z7HDp0qJx5/PhxObPbrK+vt65fu3at3OPt27flzNzcXNcjEeKNBogTGiBOaIA4oQHihAaIExogTmiAOKEB4sb6wt7r16/LmdXV1b6f8/nz53LmypUrreu3bt3q+xzDVH0wrGma5tmzZ63rXS7jTUzUf8JHjx4tZ3zYKssbDRAnNECc0ABxQgPECQ0QJzRAnNAAcUIDxI31hb1hmZqaKmf+tQt5lcuXL5cz1a9DdjEzMzOU59AfbzRAnNAAcUIDxAkNECc0QJzQAHFCA8QJDRA31hf2uvxU6uTkZOv6xsZGuceJEye6HmkknD59unX90aNH5R5bW1vlTK/X63ymv7l582bfe5DnjQaIExogTmiAOKEB4oQGiBMaIE5ogLixvkdz+/btcubdu3et69WvLTZN02xubpYzXe6mVJaWlsqZnz9/ljM/fvxoXe9y/+XIkSPlzLlz5/pab5qm2b9/fznDzvNGA8QJDRAnNECc0ABxQgPECQ0QJzRAnNAAcb0uHygasKE/sB/Ly8ut64uLi+UeXT6O9fHjx9b1YX1IqmmaZn5+vnV9enq63OPBgwflzOzsbOczMbI6/dF5owHihAaIExogTmiAOKEB4oQGiBMaIE5ogDgX9vq0vr5ezlRfrGuapllZWWldf/HiRbnH06dPy5lLly6VM6dOnWpdP3jwYLkHY8OFPWA0CA0QJzRAnNAAcUIDxAkNECc0QJx7NEA/3KMBRoPQAHFCA8QJDRAnNECc0ABxQgPECQ0QJzRAnNAAcUIDxAkNECc0QJzQAHFCA8QJDRAnNECc0ABxQgPECQ0QJzRAnNAAcUIDxAkNECc0QJzQAHFCA8QJDRAnNECc0ABxQgPECQ0QJzRAnNAAcUIDxAkNECc0QJzQAHFCA8QJDRAnNECc0ABxQgPETezAM3s78ExgB3mjAeKEBogTGiBOaIA4oQHihAaIExogTmiAOKEB4oQGiBMaIE5ogDihAeKEBogTGiBOaIA4oQHihAaIExogTmiAOKEB4oQGiBMaIE5ogDihAeL+Axg/+ff/MsX6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "some_digit = X[36000]\n",
    "some_digit_image = some_digit.reshape(28, 28)\n",
    "plt.imshow(some_digit_image, cmap = matplotlib.cm.binary,\n",
    "           interpolation=\"nearest\")\n",
    "plt.axis(\"off\")\n",
    "\n",
    "save_fig(\"some_digit_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_digit(data):\n",
    "    image = data.reshape(28, 28)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.binary,\n",
    "               interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 숫자 그림을 위한 추가 함수\n",
    "def plot_digits(instances, images_per_row=10, **options):\n",
    "    size = 28\n",
    "    images_per_row = min(len(instances), images_per_row)\n",
    "    images = [instance.reshape(size,size) for instance in instances]\n",
    "    n_rows = (len(instances) - 1) // images_per_row + 1\n",
    "    row_images = []\n",
    "    n_empty = n_rows * images_per_row - len(instances)\n",
    "    images.append(np.zeros((size, size * n_empty)))\n",
    "    for row in range(n_rows):\n",
    "        rimages = images[row * images_per_row : (row + 1) * images_per_row]\n",
    "        row_images.append(np.concatenate(rimages, axis=1))\n",
    "    image = np.concatenate(row_images, axis=0)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.binary, **options)\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 69756 characters]\n",
      "text/plain": [
       "<Figure size 648x648 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(9,9))\n",
    "example_images = np.r_[X[:12000:600], X[13000:30600:600], X[30600:60000:590]]\n",
    "plot_digits(example_images, images_per_row=10)\n",
    "save_fig(\"more_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[36000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "X_train, y_train = X_train[shuffle_index], y_train[shuffle_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 이진 분류기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_5 = (y_train == 5)\n",
    "y_test_5 = (y_test == 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "       learning_rate='optimal', loss='hinge', max_iter=5, n_iter=None,\n",
       "       n_jobs=1, penalty='l2', power_t=0.5, random_state=42, shuffle=True,\n",
       "       tol=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = SGDClassifier(max_iter=5, random_state=42)\n",
    "sgd_clf.fit(X_train, y_train_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9502 , 0.96565, 0.96495])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf, X_train, y_train_5, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9502\n",
      "0.96565\n",
      "0.96495\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.base import clone\n",
    "\n",
    "skfolds = StratifiedKFold(n_splits=3, random_state=42)\n",
    "\n",
    "for train_index, test_index in skfolds.split(X_train, y_train_5):\n",
    "    clone_clf = clone(sgd_clf)\n",
    "    X_train_folds = X_train[train_index]\n",
    "    y_train_folds = (y_train_5[train_index])\n",
    "    X_test_fold = X_train[test_index]\n",
    "    y_test_fold = (y_train_5[test_index])\n",
    "\n",
    "    clone_clf.fit(X_train_folds, y_train_folds)\n",
    "    y_pred = clone_clf.predict(X_test_fold)\n",
    "    n_correct = sum(y_pred == y_test_fold)\n",
    "    print(n_correct / len(y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "class Never5Classifier(BaseEstimator):\n",
    "    def fit(self, X, y=None):\n",
    "        pass\n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype=bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.909  , 0.90715, 0.9128 ])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "never_5_clf = Never5Classifier()\n",
    "cross_val_score(never_5_clf, X_train, y_train_5, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53272,  1307],\n",
       "       [ 1077,  4344]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_perfect_predictions = y_train_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[54579,     0],\n",
       "       [    0,  5421]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_train_5, y_train_perfect_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7687135020350381"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "precision_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7687135020350381"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4344 / (4344 + 1307)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.801328168234643"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.801328168234643"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4344 / (4344 + 1077)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7846820809248555"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7846820809248555"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4344 / (4344 + (1077 + 1307)/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([161855.74572176])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 0\n",
    "y_some_digit_pred = (y_scores > threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 200000\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3,\n",
    "                             method=\"decision_function\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "노트: 만약 사이킷런 0.19.0 버전을 사용하고 있다면 `method=\"decision_function\"` 옵션으로 `cross_val_predict()` 함수를 사용할 때 이진 분류에서 1차원 배열이 아니고 2차원 배열을 반환하는 [버그](https://github.com/scikit-learn/scikit-learn/issues/9589)가 있습니다. 사이킷런 0.19.1로 업그레이드하거나 다음 셀에서 y_scores\\[:, 1\\] 처럼 두 번째 열만 사용해야 합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hack to work around issue #9589 introduced in Scikit-Learn 0.19.0\n",
    "# if y_scores.ndim == 2:\n",
    "#     y_scores = y_scores[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 27344 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label=\"정밀도\", linewidth=2)\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label=\"재현율\", linewidth=2)\n",
    "    plt.xlabel(\"임계값\", fontsize=16)\n",
    "    plt.legend(loc=\"upper left\", fontsize=16)\n",
    "    plt.ylim([0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.xlim([-700000, 700000])\n",
    "save_fig(\"precision_recall_vs_threshold_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(y_train_pred == (y_scores > 0)).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred_90 = (y_scores > 70000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8659205116491548"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6993174691016417"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 17068 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_vs_recall(precisions, recalls):\n",
    "    plt.plot(recalls, precisions, \"b-\", linewidth=2)\n",
    "    plt.xlabel(\"재현율\", fontsize=16)\n",
    "    plt.ylabel(\"정밀도\", fontsize=16)\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_precision_vs_recall(precisions, recalls)\n",
    "save_fig(\"precision_vs_recall_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC 곡선"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 29116 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel('거짓 양성 비율', fontsize=16)\n",
    "    plt.ylabel('진짜 양성 비율', fontsize=16)\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_roc_curve(fpr, tpr)\n",
    "save_fig(\"roc_curve_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9624496555967155"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "forest_clf = RandomForestClassifier(random_state=42)\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3,\n",
    "                                    method=\"predict_proba\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores_forest = y_probas_forest[:, 1] # 점수는 양상 클래스의 확률입니다\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5,y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35876 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(fpr, tpr, \"b:\", linewidth=2, label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest, tpr_forest, \"랜덤 포레스트\")\n",
    "plt.legend(loc=\"lower right\", fontsize=16)\n",
    "save_fig(\"roc_curve_comparison_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9931243366003829"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9852973447443494"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3)\n",
    "precision_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8282604685482383"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 다중 분류"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X_train, y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-311402.62954431, -363517.28355739, -446449.5306454 ,\n",
       "        -183226.61023518, -414337.15339485,  161855.74572176,\n",
       "        -452576.39616343, -471957.14962573, -518542.33997148,\n",
       "        -536774.63961222]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(max_iter=5, random_state=42))\n",
    "ovo_clf.fit(X_train, y_train)\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovo_clf.estimators_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.1, 0. , 0. , 0.1, 0. , 0.8, 0. , 0. , 0. , 0. ]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.84063187, 0.84899245, 0.86652998])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.91011798, 0.90874544, 0.906636  ])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n",
    "cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5725,    3,   24,    9,   10,   49,   50,   10,   39,    4],\n",
       "       [   2, 6493,   43,   25,    7,   40,    5,   10,  109,    8],\n",
       "       [  51,   41, 5321,  104,   89,   26,   87,   60,  166,   13],\n",
       "       [  47,   46,  141, 5342,    1,  231,   40,   50,  141,   92],\n",
       "       [  19,   29,   41,   10, 5366,    9,   56,   37,   86,  189],\n",
       "       [  73,   45,   36,  193,   64, 4582,  111,   30,  193,   94],\n",
       "       [  29,   34,   44,    2,   42,   85, 5627,   10,   45,    0],\n",
       "       [  25,   24,   74,   32,   54,   12,    6, 5787,   15,  236],\n",
       "       [  52,  161,   73,  156,   10,  163,   61,   25, 5027,  123],\n",
       "       [  43,   35,   26,   92,  178,   28,    2,  223,   82, 5240]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(matrix):\n",
    "    \"\"\"컬러 오차 행렬을 원할 경우\"\"\"\n",
    "    fig = plt.figure(figsize=(8,8))\n",
    "    ax = fig.add_subplot(111)\n",
    "    cax = ax.matshow(matrix)\n",
    "    fig.colorbar(cax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 4188 characters]\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(conf_mx, cmap=plt.cm.gray)\n",
    "save_fig(\"confusion_matrix_plot\", tight_layout=False)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_sums = conf_mx.sum(axis=1, keepdims=True)\n",
    "norm_conf_mx = conf_mx / row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 4568 characters]\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.fill_diagonal(norm_conf_mx, 0)\n",
    "plt.matshow(norm_conf_mx, cmap=plt.cm.gray)\n",
    "save_fig(\"confusion_matrix_errors_plot\", tight_layout=False)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 71752 characters]\n",
      "text/plain": [
       "<Figure size 576x576 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cl_a, cl_b = 3, 5\n",
    "X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]\n",
    "X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]\n",
    "X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]\n",
    "X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.subplot(221); plot_digits(X_aa[:25], images_per_row=5)\n",
    "plt.subplot(222); plot_digits(X_ab[:25], images_per_row=5)\n",
    "plt.subplot(223); plot_digits(X_ba[:25], images_per_row=5)\n",
    "plt.subplot(224); plot_digits(X_bb[:25], images_per_row=5)\n",
    "save_fig(\"error_analysis_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 다중 레이블 분류"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "y_train_large = (y_train >= 7)\n",
    "y_train_odd = (y_train % 2 == 1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_multilabel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False,  True]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**경고**: 다음 셀은 실행하는데 매우 오래 걸립니다(하드웨어에 따라 몇 시간이 걸릴 수 있습니다)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97709078477525"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3, n_jobs=-1)\n",
    "f1_score(y_multilabel, y_train_knn_pred, average=\"macro\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 다중 출력 분류"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise = np.random.randint(0, 100, (len(X_train), 784))\n",
    "X_train_mod = X_train + noise\n",
    "noise = np.random.randint(0, 100, (len(X_test), 784))\n",
    "X_test_mod = X_test + noise\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 6676 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "some_index = 5500\n",
    "plt.subplot(121); plot_digit(X_test_mod[some_index])\n",
    "plt.subplot(122); plot_digit(y_test_mod[some_index])\n",
    "save_fig(\"noisy_digit_example_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARoAAAEYCAYAAACDezmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAABfFJREFUeJzt3aFvVF8agOGZTXE1OBoCJAgUGILDohAkFSgEJCQkWJL+BzgEwZFgwKGQOBQKUQXVgAAkBAKmout21d4zXfp2pv09jz1f7j2ifXPEyZ353t7eDKD0r2VvADj+hAbICQ2QExogJzRATmiAnNAAOaEBckID5NaW8E5XkeH4mC8y5EQD5IQGyAkNkBMaICc0QE5ogJzQADmhAXJCA+SEBsgJDZATGiAnNEBOaICc0AA5oQFyQgPkhAbICQ2QExogJzRATmiAnNAAOaEBckID5IQGyAkNkBMaICc0QE5ogJzQADmhAXJCA+SEBsgJDZATGiAnNEBOaICc0AA5oQFyQgPkhAbICQ2QExogJzRATmiAnNAAOaEBckID5IQGyAkNkBMaICc0QE5ogJzQADmhAXJCA+SEBsgJDZATGiC3tuwNAPuzu7s7nDlx4sQh7GRxTjRATmiAnNAAOaEBckID5IQGyAkNkBMaIOfCHvv2+fPn4czOzs5w5uzZs5Pr586dGz7j9+/fw5mNjY3hzKNHjybXt7e3h894+fLlcGZzc3Ny/c+fP8Nn3L59ezjz4MGD4cy3b9+GMwfFiQbICQ2QExogJzRATmiAnNAAOaEBcvO9vb3Dfuehv5D/ms/ny94CE5bw//i3FvqDcqIBckID5IQGyAkNkBMaICc0QE5ogJzQADkfvjpGnj59uuwt7MvW1tbk+qVLlw5pJ7PZ1atXJ9fPnz9/SDs5npxogJzQADmhAXJCA+SEBsgJDZATGiAnNEDOF/aOiEV+KfHKlSsH8q7d3d3J9bU19zz5D1/YA1aD0AA5oQFyQgPkhAbICQ2QExogJzRAzs2rFfH+/fvJ9YO6jPf8+fPhjAt5HDQnGiAnNEBOaICc0AA5oQFyQgPkhAbI+fDVinj27Nnk+r179w5pJ7PZEv4mOLp8+ApYDUID5IQGyAkNkBMaICc0QE5ogJzQADkX9lbEfL7QvadD8fjx48n169evD59x4cKFg9oOq82FPWA1CA2QExogJzRATmiAnNAAOaEBckID5FzYOyJ+/PgxnHn9+vVw5tatW3+9l52dneHM3bt3hzM3b96cXF/kq4Lr6+vDGVIu7AGrQWiAnNAAOaEBckID5IQGyAkNkHOPhsSLFy+GM3fu3Jlc39zcHD7j1atXi26Jhns0wGoQGiAnNEBOaICc0AA5oQFyQgPkhAbIubBH4sOHD8OZGzduTK5//Phx+Izt7e3hzOXLl4cz/N9c2ANWg9AAOaEBckID5IQGyAkNkBMaICc0QM6FPZbm69evk+unT5/+62fMZrPZxsbGwnti31zYA1aD0AA5oQFyQgPkhAbICQ2QExogJzRAbm3ZG+Cf6927d5Prp06dGj7DZbyjwYkGyAkNkBMaICc0QE5ogJzQADmhAXLu0ZD4/v37cObhw4eT6/fv3z+o7bBkTjRATmiAnNAAOaEBckID5IQGyAkNkBMaIOeXKknM5wv9gOGkJfxtsn9+qRJYDUID5IQGyAkNkBMaICc0QE5ogJzQADlf2FsRFy9enFx/8uTJ8BmfPn0azpw5c2bRLf1Pb968+etnzGaz2du3bw/kOaw+JxogJzRATmiAnNAAOaEBckID5IQGyLlHsyJ2dnYm169du3ZIOxnb2toazvz69Ws4s76+fhDb4QhwogFyQgPkhAbICQ2QExogJzRATmiAnNAAORf2VsToVxl3d3eHz/j58+dw5suXL8OZkydPTq4fxMez+GdxogFyQgPkhAbICQ2QExogJzRATmiAnNAAufnooljg0F8IZOaLDDnRADmhAXJCA+SEBsgJDZATGiAnNEBOaICc0AA5oQFyQgPkhAbICQ2QExogJzRATmiAnNAAOaEBckID5IQGyAkNkBMaICc0QE5ogJzQALm1JbxzoV+2A44PJxogJzRATmiAnNAAOaEBckID5IQGyAkNkBMaICc0QE5ogJzQADmhAXJCA+SEBsgJDZATGiAnNEBOaICc0AA5oQFyQgPkhAbICQ2QExog92/8o662QfNJfQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "knn_clf.fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
    "plot_digit(clean_digit)\n",
    "save_fig(\"cleaned_digit_example_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 추가 내용"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 더미 (즉, 랜덤) 분류기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.dummy import DummyClassifier\n",
    "dmy_clf = DummyClassifier()\n",
    "y_probas_dmy = cross_val_predict(dmy_clf, X_train, y_train_5, cv=3, method=\"predict_proba\")\n",
    "y_scores_dmy = y_probas_dmy[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 17420 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fprr, tprr, thresholdsr = roc_curve(y_train_5, y_scores_dmy)\n",
    "plot_roc_curve(fprr, tprr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN 분류기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=-1, n_neighbors=4, p=2,\n",
       "           weights='distance')"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf = KNeighborsClassifier(n_jobs=-1, weights='distance', n_neighbors=4)\n",
    "knn_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_knn_pred = knn_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9714"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(y_test, y_knn_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAAD+CAYAAADVndu7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAABnJJREFUeJzt3bFrU3scxuHkIg7SoWinIlgc6uIg/gmuRdx0UpfgpCjq4CRIEQTRQXCrgtJJBxHRTQdxELeiUxGlOhSEChKQIkLuX5Dv0aRNU9/nWV9Pc5b7uT84J2271+u1gEz/bfUNAFtHACCYAEAwAYBgAgDBBACCCQAEEwAIJgAQbMeoP3BhYcGrh7DJOp1O+0/+nRMABBMACCYAEEwAIJgAQDABgGACAMFG/h7AOOt0Olt9CzBSTgAQTAAgmABAMAGAYAIAwQQAggkABBMACCYAEEwAIJgAQDABgGACAMEEAIIJAAQTAAgmABBMACCYAEAwAYBgAgDBBACCCQAEEwAIJgAQTAAgmABAMAGAYAIAwQQAggkABBMACCYAEEwAIJgAQDABgGACAMEEAILt2OobgH/Vo0ePyv39+/d9t4cPHw712SsrK3/075wAIJgAQDABgGACAMEEAIIJAATzGJBY3W633N+8eVPu8/Pz5f727dtyb7fb5T4KTgAQTAAgmABAMAGAYAIAwQQAggkABPMeAFvq9+/f5b66ujrUz6+e1X/+/Lm89tWrV0N99maamprakJ/jBADBBACCCQAEEwAIJgAQTAAgmABAMO8BsKWanvPPzMyUe6/XK/dx+M59P4cOHeq7nTx5srx2bm5uQ+7BCQCCCQAEEwAIJgAQTAAgmABAMAGAYN4DYEtdvny53Jue8zftlenp6XI/c+ZMuV+9enXgzx4XTgAQTAAgmABAMAGAYAIAwQQAggkABPMeAJvu/v37fbcXL16U1w77ff7q+rW1tfLapr9ZsLy8XO6zs7PlPg6cACCYAEAwAYBgAgDBBACCCQAE8xiQoVWP+VqtVuvixYt9t58/f2707fyxX79+lfv169fLfXFxsdw/ffr01/c0ak4AEEwAIJgAQDABgGACAMEEAIIJAATzHgBDu3btWrl3u92Bf/bk5GS5T0xMlPt///X/f9z6+np57bdv38p9ZWWl3LcDJwAIJgAQTAAgmABAMAGAYAIAwQQAgnkPgKEdO3as3O/evdt3O336dHnt2bNny/3w4cPlXlldXS33ubm5cl9aWhr4s8eFEwAEEwAIJgAQTAAgmABAMAGAYAIAwbwHwNDu3Lkz1L5Ver3epu7bgRMABBMACCYAEEwAIJgAQDABgGACAMG8B7BNfP36tdx37drVd9uzZ89G384/oen7/O12e6j96dOn5d70exRGwQkAggkABBMACCYAEEwAIJgAQDCPAcfEjRs3yv3BgwflvnPnzr7b/v37y2ufPHlS7tvZ2tpa3+3KlSvltR8+fCj3mZmZQW5prDgBQDABgGACAMEEAIIJAAQTAAgmABDMewBj4t27d+W+vLw88M/+8uVLuV+6dKncb926NfBnb7amr0k/f/6879b0nH/Hjvo/j4MHD5b7OHzdt4kTAAQTAAgmABBMACCYAEAwAYBgAgDBvAcQYHJystzH+Tl/k/Pnz5d706/mrkxPT2/azx4XTgAQTAAgmABAMAGAYAIAwQQAggkABPMewJho+h3zExMT5d7tdvtuR48eHeSWRuLEiRPl/vjx43Lv9Xrl3vQnvCs3b94c+NrtwgkAggkABBMACCYAEEwAIJgAQDCPAcfE7du3y/3jx4/lXv366/X19fLapkdtTebn58v9x48ffbfv37+X1zY9xjtw4EC5nzp1aqCt1Wq1du/eXe7/AicACCYAEEwAIJgAQDABgGACAMEEAIJ5D2CbuHDhQrlXfwL85cuX5bX37t0r9838yu3s7Gy5T01Nlfvi4mK579u376/vKYkTAAQTAAgmABBMACCYAEAwAYBgAgDBvAewTRw5cqTcq2f9Td+5X1paKvfXr1+X+7Nnz8r93Llzfbfjx4+X1+7du7fcGY4TAAQTAAgmABBMACCYAEAwAYBgAgDB2k3f9d5oCwsLo/3Av9DpdLb6FmCj/NEvaXACgGACAMEEAIIJAAQTAAgmABBMACCYAEAwAYBgAgDBBACCCQAEEwAIJgAQTAAgmABAMAGAYAIAwQQAggkABBMACCYAEEwAIJgAQDABgGACAMEEAIIJAAQTAAgmABBs5H8eHBgfTgAQTAAgmABAMAGAYAIAwQQAggkABBMACCYAEEwAIJgAQDABgGACAMEEAIIJAAQTAAgmABBMACCYAEAwAYBgAgDBBACCCQAEEwAIJgAQ7H85NfU5tCUCOwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from scipy.ndimage.interpolation import shift\n",
    "def shift_digit(digit_array, dx, dy, new=0):\n",
    "    return shift(digit_array.reshape(28, 28), [dy, dx], cval=new).reshape(784)\n",
    "\n",
    "plot_digit(shift_digit(some_digit, 5, 1, new=100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((300000, 784), (300000,))"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_expanded = [X_train]\n",
    "y_train_expanded = [y_train]\n",
    "for dx, dy in ((1, 0), (-1, 0), (0, 1), (0, -1)):\n",
    "    shifted_images = np.apply_along_axis(shift_digit, axis=1, arr=X_train, dx=dx, dy=dy)\n",
    "    X_train_expanded.append(shifted_images)\n",
    "    y_train_expanded.append(y_train)\n",
    "\n",
    "X_train_expanded = np.concatenate(X_train_expanded)\n",
    "y_train_expanded = np.concatenate(y_train_expanded)\n",
    "X_train_expanded.shape, y_train_expanded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=-1, n_neighbors=4, p=2,\n",
       "           weights='distance')"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.fit(X_train_expanded, y_train_expanded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_knn_expanded_pred = knn_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9763"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_knn_expanded_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.       , 0.       , 0.5053645, 0.       , 0.       , 0.       ,\n",
       "        0.       , 0.4946355, 0.       , 0.       ]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ambiguous_digit = X_test[2589]\n",
    "knn_clf.predict_proba([ambiguous_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAAD+CAYAAADVndu7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAABvlJREFUeJzt3V9ozv0fx/F7v4yDlc1yIMTJFCvNCSeSErIkf8IcKU4kKX8O7IAD4pwckHKktKRkJ8oJjnbiwAEpTRrmwMFGjhztPrrL7+B6X2ObjdfjcfrycX2Vnn3r+l7X1TIxMfEPkOl/s30BwOwRAAgmABBMACCYAEAwAYBgAgDBBACCCQAEmzcLr+nRQ5h5LZP5Q+4AIJgAQDABgGACAMEEAIIJAAQTAAgmABBMACCYAEAwAYBgAgDBBACCCQAEEwAIJgAQTAAgmABAMAGAYAIAwQQAggkABBMACCYAEEwAIJgAQDABgGACAMEEAIIJAAQTAAgmABBMACCYAEAwAYBgAgDBBACCCQAEmzfbF0C2z58/l/uLFy/KfXBwsNwXLVrUcOvv7y/PtrW1lfvfwB0ABBMACCYAEEwAIJgAQDABgGACAMFaJiYmfvdr/vYXZGbdu3ev3N++fdtwu3XrVnl2ZGTkl67pP9X/76dPn5ZnN2/ePKXXnmUtk/lD7gAgmABAMAGAYAIAwQQAggkABPNx4ACPHj0q9wcPHpT7w4cPy73ZR3pbWib1jhSzwB0ABBMACCYAEEwAIJgAQDABgGACAME8B/CHePPmTbnfuHGj4Xbz5s3y7Pfv33/pmiZr3bp1Dbft27eXZ7u6usr93Llz5V792zo6OsqzCdwBQDABgGACAMEEAIIJAAQTAAgmABDMcwC/yejoaLlfuXKl3AcGBsr9y5cvDbcNGzaUZ9evX1/ufX195d7Z2VnuK1eubLg1+wnuZl/dPT4+Xu4HDx5suPX09JRnE7gDgGACAMEEAIIJAAQTAAgmABBMACCY5wCmyZ07d8r90qVL5T48PFzuS5YsKffbt2833I4cOVKencuePXtW7s1+3r7Z8xPp3AFAMAGAYAIAwQQAggkABBMACCYAEKyl2fuoM+C3v+B0efLkScOtt7e3PNvsO+hPnDhR7kePHi33ZcuWlftc9fz583LfuXNnube3t5d7s99T+Iu1TOYPuQOAYAIAwQQAggkABBMACCYAEMzHgX/C9evXG26tra3l2WPHjpX7hQsXfuma/nT3798v969fv5b748ePp/Ny4rgDgGACAMEEAIIJAAQTAAgmABBMACCYjwP/hA8fPjTcvn37Vp7t7u6e7sv5YwwNDTXctmzZUp7dv39/uTf7OvZgPg4M1AQAggkABBMACCYAEEwAIJgAQDDPATDj9u3b13AbHx8vzz58+LDcFy5c+EvXFMBzAEBNACCYAEAwAYBgAgDBBACCCQAE87sATNnly5fL/cGDBw23a9eulWe9zz+z3AFAMAGAYAIAwQQAggkABBMACCYAEMz3AdDU8PBwua9atarcq/f6jx8/Xp5tbW0t95n08uXLcm9vby/3V69elfuOHTt++pp+gu8DAGoCAMEEAIIJAAQTAAgmABDM24D88/r163Lv7e0t95GRkXLv6OhouM2fP788O1XV/++Wlkm9U9bQmjVryn3r1q3lfv78+Sm9fhPeBgRqAgDBBACCCQAEEwAIJgAQTAAgmK8Fnybv3r0r92bvlTfT7HmNwcHBhtunT5/Ks0NDQ+X+8ePHcm9mwYIFDbfu7u4p/d3N/m1Lly5tuHV1dZVnT506Ve5Tvfa5wB0ABBMACCYAEEwAIJgAQDABgGACAME8B/ATqp+5Pn36dHn2/fv3U3rtZs8BTPWz7ZVDhw6V++7du8t948aNDbfly5f/0jX9Z2xsrNw7Ozun9Pf/7dwBQDABgGACAMEEAIIJAAQTAAgmABDM7wL8YO3ateXe7OeiK21tbeXe399f7nfv3i336jmDxYsXl2evXr1a7nv27Cl35iS/CwDUBACCCQAEEwAIJgAQTAAgmLcBf9DsI7UrVqxouJ08ebI8u23btnLv6ekp92YOHz7ccNu1a1d59sCBA1N6beYkbwMCNQGAYAIAwQQAggkABBMACCYAEMxzAD+4ePFiuff19TXcVq9ePd2X83/OnDlT7nv37m24bdq0abovh7nPcwBATQAgmABAMAGAYAIAwQQAggkABPMcwBwxOjpa7gMDA+V+9uzZ6bwc/nyeAwBqAgDBBACCCQAEEwAIJgAQTAAgmOcA5oixsbFy7+zs/E1Xwl/CcwBATQAgmABAMAGAYAIAwQQAggkABPMcAPydPAcA1AQAggkABBMACCYAEEwAIJgAQDABgGACAMEEAIIJAAQTAAgmABBMACCYAEAwAYBgAgDBBACCCQAEEwAIJgAQTAAg2LxZeM1JfV0xMPPcAUAwAYBgAgDBBACCCQAEEwAIJgAQTAAgmABAMAGAYAIAwQQAggkABBMACCYAEEwAIJgAQDABgGACAMEEAIIJAAQTAAgmABBMACCYAECwfwEuwBgkwWeZhgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_digit(ambiguous_digit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 해답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. 97% 정확도의 MNIST 분류기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 6 candidates, totalling 30 fits\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV]  n_neighbors=3, weights=uniform, score=0.9714214297617064, total=12.0min\n",
      "[CV] n_neighbors=3, weights=uniform ..................................\n",
      "[CV]  n_neighbors=3, weights=uniform, score=0.9698458975426906, total=12.0min\n",
      "[CV] n_neighbors=3, weights=distance .................................\n",
      "[CV]  n_neighbors=3, weights=uniform, score=0.9728265399683255, total=12.0min\n",
      "[CV] n_neighbors=3, weights=distance .................................\n",
      "[CV]  n_neighbors=3, weights=uniform, score=0.9726666666666667, total=12.0min\n",
      "[CV] n_neighbors=3, weights=distance .................................\n",
      "[CV]  n_neighbors=3, weights=distance, score=0.9712619741774261, total=11.9min\n",
      "[CV] n_neighbors=3, weights=distance .................................\n",
      "[CV]  n_neighbors=3, weights=uniform, score=0.9717405801933978, total=11.9min\n",
      "[CV] n_neighbors=3, weights=distance .................................\n",
      "[CV]  n_neighbors=3, weights=distance, score=0.9725879020163306, total=12.0min\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV] .... n_neighbors=3, weights=distance, score=0.9745, total=12.0min\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV]  n_neighbors=3, weights=distance, score=0.9743269150620989, total=11.9min\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV]  n_neighbors=3, weights=distance, score=0.9724074691563854, total=11.9min\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV]  n_neighbors=4, weights=uniform, score=0.9691795085381091, total=11.9min\n",
      "[CV] n_neighbors=4, weights=uniform ..................................\n",
      "[CV]  n_neighbors=4, weights=uniform, score=0.9698383602732877, total=12.0min\n",
      "[CV] n_neighbors=4, weights=distance .................................\n",
      "[CV]  n_neighbors=4, weights=uniform, score=0.9709093940151705, total=11.9min\n",
      "[CV] n_neighbors=4, weights=distance .................................\n",
      "[CV]  n_neighbors=4, weights=uniform, score=0.9718333333333333, total=11.9min\n",
      "[CV] n_neighbors=4, weights=distance .................................\n",
      "[CV]  n_neighbors=4, weights=uniform, score=0.9680726908969657, total=11.9min\n",
      "[CV] n_neighbors=4, weights=distance .................................\n",
      "[CV]  n_neighbors=4, weights=distance, score=0.9714285714285714, total=11.9min\n",
      "[CV] n_neighbors=4, weights=distance .................................\n",
      "[CV]  n_neighbors=4, weights=distance, score=0.9729211798033661, total=11.9min\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV]  n_neighbors=4, weights=distance, score=0.9745833333333334, total=11.9min\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV]  n_neighbors=4, weights=distance, score=0.9752438109527382, total=11.9min\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV]  n_neighbors=4, weights=distance, score=0.9720740246748917, total=11.9min\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV]  n_neighbors=5, weights=uniform, score=0.9699216797200466, total=11.9min\n",
      "[CV] n_neighbors=5, weights=uniform ..................................\n",
      "[CV]  n_neighbors=5, weights=uniform, score=0.9693461057892545, total=12.0min\n",
      "[CV] n_neighbors=5, weights=distance .................................\n",
      "[CV]  n_neighbors=5, weights=uniform, score=0.9713261648745519, total=11.9min\n",
      "[CV] n_neighbors=5, weights=distance .................................\n",
      "[CV]  n_neighbors=5, weights=uniform, score=0.9726666666666667, total=11.9min\n",
      "[CV] n_neighbors=5, weights=distance .................................\n",
      "[CV]  n_neighbors=5, weights=uniform, score=0.969406468822941, total=11.9min\n",
      "[CV] n_neighbors=5, weights=distance .................................\n",
      "[CV]  n_neighbors=5, weights=distance, score=0.9712547908681887, total=11.9min\n",
      "[CV] n_neighbors=5, weights=distance .................................\n",
      "[CV]  n_neighbors=5, weights=distance, score=0.9711786755518534, total=12.0min\n",
      "[CV]  n_neighbors=5, weights=distance, score=0.9738333333333333, total=11.9min\n",
      "[CV]  n_neighbors=5, weights=distance, score=0.9724931232808202, total=11.2min\n",
      "[CV]  n_neighbors=5, weights=distance, score=0.9714904968322774, total=11.1min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  30 out of  30 | elapsed: 466.1min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid=[{'n_neighbors': [3, 4, 5], 'weights': ['uniform', 'distance']}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=3)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = [{'weights': [\"uniform\", \"distance\"], 'n_neighbors': [3, 4, 5]}]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "grid_search = GridSearchCV(knn_clf, param_grid, cv=5, verbose=3, n_jobs=-1)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 4, 'weights': 'distance'}"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97325"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9714"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = grid_search.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 데이터 증식"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.ndimage.interpolation import shift"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shift_image(image, dx, dy):\n",
    "    image = image.reshape((28, 28))\n",
    "    shifted_image = shift(image, [dy, dx], cval=0, mode=\"constant\")\n",
    "    return shifted_image.reshape([-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10472 characters]\n",
      "text/plain": [
       "<Figure size 864x216 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "image = X_train[1000]\n",
    "shifted_image_down = shift_image(image, 0, 5)\n",
    "shifted_image_left = shift_image(image, -5, 0)\n",
    "\n",
    "plt.figure(figsize=(12,3))\n",
    "plt.subplot(131)\n",
    "plt.title(\"Original\", fontsize=14)\n",
    "plt.imshow(image.reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\n",
    "plt.subplot(132)\n",
    "plt.title(\"Shifted down\", fontsize=14)\n",
    "plt.imshow(shifted_image_down.reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\n",
    "plt.subplot(133)\n",
    "plt.title(\"Shifted left\", fontsize=14)\n",
    "plt.imshow(shifted_image_left.reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_augmented = [image for image in X_train]\n",
    "y_train_augmented = [label for label in y_train]\n",
    "\n",
    "for dx, dy in ((1, 0), (-1, 0), (0, 1), (0, -1)):\n",
    "    for image, label in zip(X_train, y_train):\n",
    "        X_train_augmented.append(shift_image(image, dx, dy))\n",
    "        y_train_augmented.append(label)\n",
    "\n",
    "X_train_augmented = np.array(X_train_augmented)\n",
    "y_train_augmented = np.array(y_train_augmented)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffle_idx = np.random.permutation(len(X_train_augmented))\n",
    "X_train_augmented = X_train_augmented[shuffle_idx]\n",
    "y_train_augmented = y_train_augmented[shuffle_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_clf = KNeighborsClassifier(**grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=4, p=2,\n",
       "           weights='distance')"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_clf.fit(X_train_augmented, y_train_augmented)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9763"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = knn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "간단한 데이터 증식으로 0.5%의 정확도가 향상되었습니다. :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. 타이타닉 데이터셋에 도전"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "승객의 나이, 성별, 승객 등급, 승선 위치 같은 속성을 기반으로 하여 승객의 생존 여부를 예측하는 것이 목표입니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터를 적재합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "TITANIC_PATH = os.path.join(\"datasets\", \"titanic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def load_titanic_data(filename, titanic_path=TITANIC_PATH):\n",
    "    csv_path = os.path.join(titanic_path, filename)\n",
    "    return pd.read_csv(csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = load_titanic_data(\"train.csv\")\n",
    "test_data = load_titanic_data(\"test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터는 이미 훈련 세트와 테스트 세트로 분리되어 있습니다. 그러나 테스트 데이터는 레이블을 가지고 있지 않습니다: 훈련 데이터를 이용하여 가능한 최고의 모델을 만들고 테스트 데이터에 대한 예측을 캐글(Kaggle)에 업로드하여 최종 점수를 확인하는 것이 목표입니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훈련 세트에서 맨 위 몇 개의 열을 살펴 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "속성은 다음과 같은 의미를 가집니다:\n",
    "* **Survived**: 타깃입니다. 0은 생존하지 못한 것이고 1은 생존을 의미합니다.\n",
    "* **Pclass**: 승객 등급. 1, 2, 3등석.\n",
    "* **Name**, **Sex**, **Age**: 이름 그대로 의미입니다.\n",
    "* **SibSp**: 함께 탑승한 형제, 배우자의 수.\n",
    "* **Parch**: 함께 탑승한 자녀, 부모의 수.\n",
    "* **Ticket**: 티켓 아이디\n",
    "* **Fare**: 티켓 요금 (파운드)\n",
    "* **Cabin**: 객실 번호\n",
    "* **Embarked**: 승객이 탑승한 곳. C(Cherbourg), Q(Queenstown), S(Southampton)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "누락된 데이터가 얼마나 되는지 알아보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 12 columns):\n",
      "PassengerId    891 non-null int64\n",
      "Survived       891 non-null int64\n",
      "Pclass         891 non-null int64\n",
      "Name           891 non-null object\n",
      "Sex            891 non-null object\n",
      "Age            714 non-null float64\n",
      "SibSp          891 non-null int64\n",
      "Parch          891 non-null int64\n",
      "Ticket         891 non-null object\n",
      "Fare           891 non-null float64\n",
      "Cabin          204 non-null object\n",
      "Embarked       889 non-null object\n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 83.6+ KB\n"
     ]
    }
   ],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "괜찮네요. **Age**, **Cabin**, **Embarked** 속성의 일부가 null입니다(891개의 non-null 보다 작습니다). 특히 **Cabin**은 77%가 null입니다. 일단 **Cabin**은 무시하고 나머지를 활용하겠습니다. **Age**는 19%가 null이므로 이를 어떻게 처리할지 결정해야 합니다. null을 중간 나이로 바꾸는 것이 괜찮아 보입니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Name**과 **Ticket** 속성도 값을 가지고 있지만 머신러닝 모델이 사용할 수 있는 숫자로 변환하는 것이 조금 까다롭습니다. 그래서 지금은 이 두 속성을 무시하겠습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "통계치를 살펴 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>714.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.383838</td>\n",
       "      <td>2.308642</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.523008</td>\n",
       "      <td>0.381594</td>\n",
       "      <td>32.204208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>257.353842</td>\n",
       "      <td>0.486592</td>\n",
       "      <td>0.836071</td>\n",
       "      <td>14.526497</td>\n",
       "      <td>1.102743</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>49.693429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>223.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.125000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>668.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>512.329200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
       "count   891.000000  891.000000  891.000000  714.000000  891.000000   \n",
       "mean    446.000000    0.383838    2.308642   29.699118    0.523008   \n",
       "std     257.353842    0.486592    0.836071   14.526497    1.102743   \n",
       "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
       "25%     223.500000    0.000000    2.000000   20.125000    0.000000   \n",
       "50%     446.000000    0.000000    3.000000   28.000000    0.000000   \n",
       "75%     668.500000    1.000000    3.000000   38.000000    1.000000   \n",
       "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
       "\n",
       "            Parch        Fare  \n",
       "count  891.000000  891.000000  \n",
       "mean     0.381594   32.204208  \n",
       "std      0.806057   49.693429  \n",
       "min      0.000000    0.000000  \n",
       "25%      0.000000    7.910400  \n",
       "50%      0.000000   14.454200  \n",
       "75%      0.000000   31.000000  \n",
       "max      6.000000  512.329200  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 이크, 38%만 **Survived**입니다. :( 거의 40%에 가까우므로 정확도를 사용해 모델을 평가해도 괜찮을 것 같습니다.\n",
    "* 평균 **Fare**는 32.20 파운드라 그렇게 비싸보이지는 않습니다(아마 요금을 많이 반환해 주었기 때문일 것입니다)\n",
    "* 평균 **Age**는 30보다 작습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "타깃이 0과 1로 이루어졌는지 확인합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    549\n",
       "1    342\n",
       "Name: Survived, dtype: int64"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Survived\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "범주형 특성들을 확인해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    491\n",
       "1    216\n",
       "2    184\n",
       "Name: Pclass, dtype: int64"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Pclass\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male      577\n",
       "female    314\n",
       "Name: Sex, dtype: int64"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Sex\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "S    644\n",
       "C    168\n",
       "Q     77\n",
       "Name: Embarked, dtype: int64"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"Embarked\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Embarked** 특성은 승객이 탑승한 곳을 알려 줍니다: C=Cherbourg, Q=Queenstown, S=Southampton."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`CategoricalEncoder` 클래스는 범주형 특성을 원-핫 벡터로 변환시켜 줍니다. 이 클래스는 사이킷런 0.20에 포함될 예정이지만 지금은 아래 코드를 사용합니다([#9151](https://github.com/scikit-learn/scikit-learn/pull/9151) 풀 리퀘스트에서 복사했습니다)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PR #9151에서 가져온 CategoricalEncoder 클래스의 정의.\n",
    "# 이 클래스는 사이킷런 0.20에 포함될 예정입니다.\n",
    "# 이 셀을 실행하거나 복사해서 사용하세요. 이 코드를 모두 이해할 필요는 없습니다.\n",
    "\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.utils import check_array\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from scipy import sparse\n",
    "\n",
    "class CategoricalEncoder(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"Encode categorical features as a numeric array.\n",
    "    The input to this transformer should be a matrix of integers or strings,\n",
    "    denoting the values taken on by categorical (discrete) features.\n",
    "    The features can be encoded using a one-hot aka one-of-K scheme\n",
    "    (``encoding='onehot'``, the default) or converted to ordinal integers\n",
    "    (``encoding='ordinal'``).\n",
    "    This encoding is needed for feeding categorical data to many scikit-learn\n",
    "    estimators, notably linear models and SVMs with the standard kernels.\n",
    "    Read more in the :ref:`User Guide <preprocessing_categorical_features>`.\n",
    "    Parameters\n",
    "    ----------\n",
    "    encoding : str, 'onehot', 'onehot-dense' or 'ordinal'\n",
    "        The type of encoding to use (default is 'onehot'):\n",
    "        - 'onehot': encode the features using a one-hot aka one-of-K scheme\n",
    "          (or also called 'dummy' encoding). This creates a binary column for\n",
    "          each category and returns a sparse matrix.\n",
    "        - 'onehot-dense': the same as 'onehot' but returns a dense array\n",
    "          instead of a sparse matrix.\n",
    "        - 'ordinal': encode the features as ordinal integers. This results in\n",
    "          a single column of integers (0 to n_categories - 1) per feature.\n",
    "    categories : 'auto' or a list of lists/arrays of values.\n",
    "        Categories (unique values) per feature:\n",
    "        - 'auto' : Determine categories automatically from the training data.\n",
    "        - list : ``categories[i]`` holds the categories expected in the ith\n",
    "          column. The passed categories are sorted before encoding the data\n",
    "          (used categories can be found in the ``categories_`` attribute).\n",
    "    dtype : number type, default np.float64\n",
    "        Desired dtype of output.\n",
    "    handle_unknown : 'error' (default) or 'ignore'\n",
    "        Whether to raise an error or ignore if a unknown categorical feature is\n",
    "        present during transform (default is to raise). When this is parameter\n",
    "        is set to 'ignore' and an unknown category is encountered during\n",
    "        transform, the resulting one-hot encoded columns for this feature\n",
    "        will be all zeros.\n",
    "        Ignoring unknown categories is not supported for\n",
    "        ``encoding='ordinal'``.\n",
    "    Attributes\n",
    "    ----------\n",
    "    categories_ : list of arrays\n",
    "        The categories of each feature determined during fitting. When\n",
    "        categories were specified manually, this holds the sorted categories\n",
    "        (in order corresponding with output of `transform`).\n",
    "    Examples\n",
    "    --------\n",
    "    Given a dataset with three features and two samples, we let the encoder\n",
    "    find the maximum value per feature and transform the data to a binary\n",
    "    one-hot encoding.\n",
    "    >>> from sklearn.preprocessing import CategoricalEncoder\n",
    "    >>> enc = CategoricalEncoder(handle_unknown='ignore')\n",
    "    >>> enc.fit([[0, 0, 3], [1, 1, 0], [0, 2, 1], [1, 0, 2]])\n",
    "    ... # doctest: +ELLIPSIS\n",
    "    CategoricalEncoder(categories='auto', dtype=<... 'numpy.float64'>,\n",
    "              encoding='onehot', handle_unknown='ignore')\n",
    "    >>> enc.transform([[0, 1, 1], [1, 0, 4]]).toarray()\n",
    "    array([[ 1.,  0.,  0.,  1.,  0.,  0.,  1.,  0.,  0.],\n",
    "           [ 0.,  1.,  1.,  0.,  0.,  0.,  0.,  0.,  0.]])\n",
    "    See also\n",
    "    --------\n",
    "    sklearn.preprocessing.OneHotEncoder : performs a one-hot encoding of\n",
    "      integer ordinal features. The ``OneHotEncoder assumes`` that input\n",
    "      features take on values in the range ``[0, max(feature)]`` instead of\n",
    "      using the unique values.\n",
    "    sklearn.feature_extraction.DictVectorizer : performs a one-hot encoding of\n",
    "      dictionary items (also handles string-valued features).\n",
    "    sklearn.feature_extraction.FeatureHasher : performs an approximate one-hot\n",
    "      encoding of dictionary items or strings.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, encoding='onehot', categories='auto', dtype=np.float64,\n",
    "                 handle_unknown='error'):\n",
    "        self.encoding = encoding\n",
    "        self.categories = categories\n",
    "        self.dtype = dtype\n",
    "        self.handle_unknown = handle_unknown\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        \"\"\"Fit the CategoricalEncoder to X.\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : array-like, shape [n_samples, n_feature]\n",
    "            The data to determine the categories of each feature.\n",
    "        Returns\n",
    "        -------\n",
    "        self\n",
    "        \"\"\"\n",
    "\n",
    "        if self.encoding not in ['onehot', 'onehot-dense', 'ordinal']:\n",
    "            template = (\"encoding should be either 'onehot', 'onehot-dense' \"\n",
    "                        \"or 'ordinal', got %s\")\n",
    "            raise ValueError(template % self.handle_unknown)\n",
    "\n",
    "        if self.handle_unknown not in ['error', 'ignore']:\n",
    "            template = (\"handle_unknown should be either 'error' or \"\n",
    "                        \"'ignore', got %s\")\n",
    "            raise ValueError(template % self.handle_unknown)\n",
    "\n",
    "        if self.encoding == 'ordinal' and self.handle_unknown == 'ignore':\n",
    "            raise ValueError(\"handle_unknown='ignore' is not supported for\"\n",
    "                             \" encoding='ordinal'\")\n",
    "\n",
    "        X = check_array(X, dtype=np.object, accept_sparse='csc', copy=True)\n",
    "        n_samples, n_features = X.shape\n",
    "\n",
    "        self._label_encoders_ = [LabelEncoder() for _ in range(n_features)]\n",
    "\n",
    "        for i in range(n_features):\n",
    "            le = self._label_encoders_[i]\n",
    "            Xi = X[:, i]\n",
    "            if self.categories == 'auto':\n",
    "                le.fit(Xi)\n",
    "            else:\n",
    "                valid_mask = np.in1d(Xi, self.categories[i])\n",
    "                if not np.all(valid_mask):\n",
    "                    if self.handle_unknown == 'error':\n",
    "                        diff = np.unique(Xi[~valid_mask])\n",
    "                        msg = (\"Found unknown categories {0} in column {1}\"\n",
    "                               \" during fit\".format(diff, i))\n",
    "                        raise ValueError(msg)\n",
    "                le.classes_ = np.array(np.sort(self.categories[i]))\n",
    "\n",
    "        self.categories_ = [le.classes_ for le in self._label_encoders_]\n",
    "\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        \"\"\"Transform X using one-hot encoding.\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : array-like, shape [n_samples, n_features]\n",
    "            The data to encode.\n",
    "        Returns\n",
    "        -------\n",
    "        X_out : sparse matrix or a 2-d array\n",
    "            Transformed input.\n",
    "        \"\"\"\n",
    "        X = check_array(X, accept_sparse='csc', dtype=np.object, copy=True)\n",
    "        n_samples, n_features = X.shape\n",
    "        X_int = np.zeros_like(X, dtype=np.int)\n",
    "        X_mask = np.ones_like(X, dtype=np.bool)\n",
    "\n",
    "        for i in range(n_features):\n",
    "            valid_mask = np.in1d(X[:, i], self.categories_[i])\n",
    "\n",
    "            if not np.all(valid_mask):\n",
    "                if self.handle_unknown == 'error':\n",
    "                    diff = np.unique(X[~valid_mask, i])\n",
    "                    msg = (\"Found unknown categories {0} in column {1}\"\n",
    "                           \" during transform\".format(diff, i))\n",
    "                    raise ValueError(msg)\n",
    "                else:\n",
    "                    # Set the problematic rows to an acceptable value and\n",
    "                    # continue `The rows are marked `X_mask` and will be\n",
    "                    # removed later.\n",
    "                    X_mask[:, i] = valid_mask\n",
    "                    X[:, i][~valid_mask] = self.categories_[i][0]\n",
    "            X_int[:, i] = self._label_encoders_[i].transform(X[:, i])\n",
    "\n",
    "        if self.encoding == 'ordinal':\n",
    "            return X_int.astype(self.dtype, copy=False)\n",
    "\n",
    "        mask = X_mask.ravel()\n",
    "        n_values = [cats.shape[0] for cats in self.categories_]\n",
    "        n_values = np.array([0] + n_values)\n",
    "        indices = np.cumsum(n_values)\n",
    "\n",
    "        column_indices = (X_int + indices[:-1]).ravel()[mask]\n",
    "        row_indices = np.repeat(np.arange(n_samples, dtype=np.int32),\n",
    "                                n_features)[mask]\n",
    "        data = np.ones(n_samples * n_features)[mask]\n",
    "\n",
    "        out = sparse.csc_matrix((data, (row_indices, column_indices)),\n",
    "                                shape=(n_samples, indices[-1]),\n",
    "                                dtype=self.dtype).tocsr()\n",
    "        if self.encoding == 'onehot-dense':\n",
    "            return out.toarray()\n",
    "        else:\n",
    "            return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 전처리 파이프라인을 만듭니다. `DataFrame`으로부터 특정 속성만 선택하기 위해 이전 장에서 만든 `DataframeSelector`를 재사용하겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "# 사이킷런이 DataFrame을 바로 사용하지 못하므로\n",
    "# 수치형이나 범주형 컬럼을 선택하는 클래스를 만듭니다.\n",
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "숫자 특성을 위한 파이프라인을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import Imputer\n",
    "\n",
    "imputer = Imputer(strategy=\"median\")\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "        (\"select_numeric\", DataFrameSelector([\"Age\", \"SibSp\", \"Parch\", \"Fare\"])),\n",
    "        (\"imputer\", Imputer(strategy=\"median\")),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[22.    ,  1.    ,  0.    ,  7.25  ],\n",
       "       [38.    ,  1.    ,  0.    , 71.2833],\n",
       "       [26.    ,  0.    ,  0.    ,  7.925 ],\n",
       "       ...,\n",
       "       [28.    ,  1.    ,  2.    , 23.45  ],\n",
       "       [26.    ,  0.    ,  0.    , 30.    ],\n",
       "       [32.    ,  0.    ,  0.    ,  7.75  ]])"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_pipeline.fit_transform(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "문자열로된 범주형 열을 위해 별도의 Imputer 클래스가 필요합니다(일반 `Imputer` 클래스는 이를 처리하지 못합니다):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stackoverflow.com/questions/25239958 에서 착안했습니다\n",
    "class MostFrequentImputer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        self.most_frequent = pd.Series([X[c].value_counts().index[0] for c in X],\n",
    "                                       index=X.columns)\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        return X.fillna(self.most_frequent)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 범주형 특성을 위한 파이프라인을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_pipeline = Pipeline([\n",
    "        (\"select_cat\", DataFrameSelector([\"Pclass\", \"Sex\", \"Embarked\"])),\n",
    "        (\"imputer\", MostFrequentImputer()),\n",
    "        (\"cat_encoder\", CategoricalEncoder(encoding='onehot-dense')),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 1., ..., 0., 0., 1.],\n",
       "       [1., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 1., ..., 0., 0., 1.],\n",
       "       ...,\n",
       "       [0., 0., 1., ..., 0., 0., 1.],\n",
       "       [1., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 1., ..., 0., 1., 0.]])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_pipeline.fit_transform(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "마지막으로 숫자와 범주형 파이프라인을 연결합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import FeatureUnion\n",
    "preprocess_pipeline = FeatureUnion(transformer_list=[\n",
    "        (\"num_pipeline\", num_pipeline),\n",
    "        (\"cat_pipeline\", cat_pipeline),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋습니다! 이제 원본 데이터를 받아 머신러닝 모델에 주입할 숫자 입력 특성을 출력하는 전처리 파이프라인을 만들었습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[22.,  1.,  0., ...,  0.,  0.,  1.],\n",
       "       [38.,  1.,  0., ...,  1.,  0.,  0.],\n",
       "       [26.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       ...,\n",
       "       [28.,  1.,  2., ...,  0.,  0.,  1.],\n",
       "       [26.,  0.,  0., ...,  1.,  0.,  0.],\n",
       "       [32.,  0.,  0., ...,  0.,  1.,  0.]])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = preprocess_pipeline.fit_transform(train_data)\n",
    "X_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "레이블을 가져옵니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train_data[\"Survived\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 분류기를 훈련시킬 차례입니다. 먼저 `SVC`를 사용해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "svm_clf = SVC()\n",
    "svm_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "모델이 잘 훈련된 것 같습니다. 이를 사용해서 테스트 세트에 대한 예측을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = preprocess_pipeline.transform(test_data)\n",
    "y_pred = svm_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 예측 결과를 (캐글에서 기대하는 형태인) CSV 파일로 만들어 업로드하고 평가를 받아볼 수 있습니다. 하지만 그냥 좋을거라 기대하는 것보다 교차 검증으로 모델이 얼마나 좋은지 평가하는 것이 좋습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7365250822835092"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "scores = cross_val_score(svm_clf, X_train, y_train, cv=10)\n",
    "scores.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "정확도가 73% 이상입니다. 확실히 무작위로 선택한 것보다는 좋습니다. 하지만 아주 높은 점수는 아닙니다. 캐글에서 타이타닉 경연 대회의 [리더보드](https://www.kaggle.com/c/titanic/leaderboard)를 보면 상위 10%내에 들려면 80% 이상의 정확도를 내야합니다. 어떤 사람들은 100%를 달성했습니다. 하지만 타이타닉의 [희생자 목록](https://www.encyclopedia-titanica.org/titanic-victims/)을 쉽게 찾을 수 있으므로 머신러닝을 사용하지 않고도 이런 정확도를 달성할 수 있습니다! ;-) 우리는 80% 정도의 정확도를 내는 모델을 만들어 보겠습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`RandomForestClassifier`를 적용해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8115690614005221"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf = RandomForestClassifier(random_state=42)\n",
    "scores = cross_val_score(forest_clf, X_train, y_train, cv=10)\n",
    "scores.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훨씬 좋네요!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 결과를 더 향상시키려면:\n",
    "* 교차 검증과 그리드 탐색을 사용하여 더 많은 모델을 비교하고 하이퍼파라미터를 튜닝하세요.\n",
    "* 특성 공학을 더 시도해 보세요, 예를 들면:\n",
    "  * **SibSp**와 **Parch**을 이 두 특성의 합으로 바꿉니다.\n",
    "  * **Survived** 특성과 관련된 이름을 구별해 보세요(가령, 이름에 \"Countess\"가 있는 경우 생존할 가능성이 높습니다).\n",
    "* 수치 특성을 범주형 특성으로 바꾸어 보세요: 예를 들어, 나이대가 다른 경우 다른 생존 비율을 가질 수 있습니다(아래 참조). 그러므로 나이 구간을 범주로 만들어 나이 대신 사용하는 것이 도움이 될 수 있스니다. 비슷하게 생존자의 30%가 혼자 여행하는 사람이기 때문에 이들을 위한 특별한 범주를 만드는 것이 도움이 될 수 있습니다(아래 참조)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AgeBucket</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>0.576923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15.0</th>\n",
       "      <td>0.362745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30.0</th>\n",
       "      <td>0.423256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45.0</th>\n",
       "      <td>0.404494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60.0</th>\n",
       "      <td>0.240000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75.0</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Survived\n",
       "AgeBucket          \n",
       "0.0        0.576923\n",
       "15.0       0.362745\n",
       "30.0       0.423256\n",
       "45.0       0.404494\n",
       "60.0       0.240000\n",
       "75.0       1.000000"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"AgeBucket\"] = train_data[\"Age\"] // 15 * 15\n",
    "train_data[[\"AgeBucket\", \"Survived\"]].groupby(['AgeBucket']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RelativesOnboard</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.303538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.552795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.578431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.724138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.136364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Survived\n",
       "RelativesOnboard          \n",
       "0                 0.303538\n",
       "1                 0.552795\n",
       "2                 0.578431\n",
       "3                 0.724138\n",
       "4                 0.200000\n",
       "5                 0.136364\n",
       "6                 0.333333\n",
       "7                 0.000000\n",
       "10                0.000000"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[\"RelativesOnboard\"] = train_data[\"SibSp\"] + train_data[\"Parch\"]\n",
    "train_data[[\"RelativesOnboard\", \"Survived\"]].groupby(['RelativesOnboard']).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. 스팸 필터"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저 데이터를 다운받습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "from six.moves import urllib\n",
    "\n",
    "DOWNLOAD_ROOT = \"http://spamassassin.apache.org/old/publiccorpus/\"\n",
    "HAM_URL = DOWNLOAD_ROOT + \"20030228_easy_ham.tar.bz2\"\n",
    "SPAM_URL = DOWNLOAD_ROOT + \"20030228_spam.tar.bz2\"\n",
    "SPAM_PATH = os.path.join(\"datasets\", \"spam\")\n",
    "\n",
    "def fetch_spam_data(spam_url=SPAM_URL, spam_path=SPAM_PATH):\n",
    "    if not os.path.isdir(spam_path):\n",
    "        os.makedirs(spam_path)\n",
    "    for filename, url in ((\"ham.tar.bz2\", HAM_URL), (\"spam.tar.bz2\", SPAM_URL)):\n",
    "        path = os.path.join(spam_path, filename)\n",
    "        if not os.path.isfile(path):\n",
    "            urllib.request.urlretrieve(url, path)\n",
    "        tar_bz2_file = tarfile.open(path)\n",
    "        tar_bz2_file.extractall(path=SPAM_PATH)\n",
    "        tar_bz2_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_spam_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "다음, 모든 이메일을 읽어 들입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "HAM_DIR = os.path.join(SPAM_PATH, \"easy_ham\")\n",
    "SPAM_DIR = os.path.join(SPAM_PATH, \"spam\")\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name) > 20]\n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name) > 20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2500"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ham_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(spam_filenames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬의 `email` 모듈을 사용해 이메일을 파싱합니다(헤더, 인코딩 등을 처리합니다):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "import email\n",
    "import email.policy\n",
    "\n",
    "def load_email(is_spam, filename, spam_path=SPAM_PATH):\n",
    "    directory = \"spam\" if is_spam else \"easy_ham\"\n",
    "    with open(os.path.join(spam_path, directory, filename), \"rb\") as f:\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "ham_emails = [load_email(is_spam=False, filename=name) for name in ham_filenames]\n",
    "spam_emails = [load_email(is_spam=True, filename=name) for name in spam_filenames]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터가 어떻게 구성되어 있는지 감을 잡기 위해 햄 메일과 스팸 메일을 하나씩 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Martin A posted:\n",
      "Tassos Papadopoulos, the Greek sculptor behind the plan, judged that the\n",
      " limestone of Mount Kerdylio, 70 miles east of Salonika and not far from the\n",
      " Mount Athos monastic community, was ideal for the patriotic sculpture. \n",
      " \n",
      " As well as Alexander's granite features, 240 ft high and 170 ft wide, a\n",
      " museum, a restored amphitheatre and car park for admiring crowds are\n",
      "planned\n",
      "---------------------\n",
      "So is this mountain limestone or granite?\n",
      "If it's limestone, it'll weather pretty fast.\n",
      "\n",
      "------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "4 DVDs Free +s&p Join Now\n",
      "http://us.click.yahoo.com/pt6YBB/NXiEAA/mG3HAA/7gSolB/TM\n",
      "---------------------------------------------------------------------~->\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "forteana-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n"
     ]
    }
   ],
   "source": [
    "print(ham_emails[1].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help wanted.  We are a 14 year old fortune 500 company, that is\n",
      "growing at a tremendous rate.  We are looking for individuals who\n",
      "want to work from home.\n",
      "\n",
      "This is an opportunity to make an excellent income.  No experience\n",
      "is required.  We will train you.\n",
      "\n",
      "So if you are looking to be employed from home with a career that has\n",
      "vast opportunities, then go:\n",
      "\n",
      "http://www.basetel.com/wealthnow\n",
      "\n",
      "We are looking for energetic and self motivated people.  If that is you\n",
      "than click on the link and fill out the form, and one of our\n",
      "employement specialist will contact you.\n",
      "\n",
      "To be removed from our link simple go to:\n",
      "\n",
      "http://www.basetel.com/remove.html\n",
      "\n",
      "\n",
      "4139vOLW7-758DoDY1425FRhM1-764SMFc8513fCsLl40\n"
     ]
    }
   ],
   "source": [
    "print(spam_emails[6].get_content().strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "어떤 이메일은 이미지나 첨부 파일을 가진 멀티파트(multipart)입니다(메일에 포함되어 있을수 있습니다). 어떤 파일들이 있는지 살펴 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_email_structure(email):\n",
    "    if isinstance(email, str):\n",
    "        return email\n",
    "    payload = email.get_payload()\n",
    "    if isinstance(payload, list):\n",
    "        return \"multipart({})\".format(\", \".join([\n",
    "            get_email_structure(sub_email)\n",
    "            for sub_email in payload\n",
    "        ]))\n",
    "    else:\n",
    "        return email.get_content_type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def structures_counter(emails):\n",
    "    structures = Counter()\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] += 1\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 2408),\n",
       " ('multipart(text/plain, application/pgp-signature)', 66),\n",
       " ('multipart(text/plain, text/html)', 8),\n",
       " ('multipart(text/plain, text/plain)', 4),\n",
       " ('multipart(text/plain)', 3),\n",
       " ('multipart(text/plain, application/octet-stream)', 2),\n",
       " ('multipart(multipart(text/plain, text/plain, text/plain), application/pgp-signature)',\n",
       "  1),\n",
       " ('multipart(text/plain, text/enriched)', 1),\n",
       " ('multipart(text/plain, application/x-java-applet)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain))', 1),\n",
       " ('multipart(text/plain, application/x-pkcs7-signature)', 1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), multipart(multipart(text/plain, application/x-pkcs7-signature)))',\n",
       "  1),\n",
       " ('multipart(text/plain, multipart(text/plain, text/plain), text/rfc822-headers)',\n",
       "  1),\n",
       " ('multipart(text/plain, video/mng)', 1),\n",
       " ('multipart(text/plain, application/ms-tnef, text/plain)', 1)]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(ham_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 218),\n",
       " ('text/html', 183),\n",
       " ('multipart(text/plain, text/html)', 45),\n",
       " ('multipart(text/html)', 20),\n",
       " ('multipart(text/plain)', 19),\n",
       " ('multipart(multipart(text/html))', 5),\n",
       " ('multipart(text/plain, image/jpeg)', 3),\n",
       " ('multipart(text/html, application/octet-stream)', 2),\n",
       " ('multipart(text/html, text/plain)', 1),\n",
       " ('multipart(text/plain, application/octet-stream)', 1),\n",
       " ('multipart/alternative', 1),\n",
       " ('multipart(multipart(text/plain, text/html), image/gif)', 1),\n",
       " ('multipart(multipart(text/html), application/octet-stream, image/jpeg)', 1)]"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(spam_emails).most_common()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "햄 메일은 평범한 텍스트가 많고 스팸은 HTML일 경우가 많습니다. 적은 수의 햄 이메일이 PGP로 서명되어 있지만 스팸 메일에는 없습니다. 요약하면 이메일 구조는 유용한 정보입니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 이메일 헤더를 살펴보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Return-Path : <12a1mailbot1@web.de>\n",
      "Delivered-To : zzzz@localhost.spamassassin.taint.org\n",
      "Received : from localhost (localhost [127.0.0.1])\tby phobos.labs.spamassassin.taint.org (Postfix) with ESMTP id 136B943C32\tfor <zzzz@localhost>; Thu, 22 Aug 2002 08:17:21 -0400 (EDT)\n",
      "Received : from mail.webnote.net [193.120.211.219]\tby localhost with POP3 (fetchmail-5.9.0)\tfor zzzz@localhost (single-drop); Thu, 22 Aug 2002 13:17:21 +0100 (IST)\n",
      "Received : from dd_it7 ([210.97.77.167])\tby webnote.net (8.9.3/8.9.3) with ESMTP id NAA04623\tfor <zzzz@spamassassin.taint.org>; Thu, 22 Aug 2002 13:09:41 +0100\n",
      "From : 12a1mailbot1@web.de\n",
      "Received : from r-smtp.korea.com - 203.122.2.197 by dd_it7  with Microsoft SMTPSVC(5.5.1775.675.6);\t Sat, 24 Aug 2002 09:42:10 +0900\n",
      "To : dcek1a1@netsgo.com\n",
      "Subject : Life Insurance - Why Pay More?\n",
      "Date : Wed, 21 Aug 2002 20:31:57 -1600\n",
      "MIME-Version : 1.0\n",
      "Message-ID : <0103c1042001882DD_IT7@dd_it7>\n",
      "Content-Type : text/html; charset=\"iso-8859-1\"\n",
      "Content-Transfer-Encoding : quoted-printable\n"
     ]
    }
   ],
   "source": [
    "for header, value in spam_emails[0].items():\n",
    "    print(header,\":\",value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "보낸사람의 이메일 주소와 같이 헤더에는 유용한 정보가 많이 있지만 여기서는 `Subject` 헤더만 다뤄 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Life Insurance - Why Pay More?'"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_emails[0][\"Subject\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋습니다. 데이터에를 더 살펴보기 전에 훈련 세트와 테스트 세트로 나누도록 하겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = np.array(ham_emails + spam_emails)\n",
    "y = np.array([0] * len(ham_emails) + [1] * len(spam_emails))\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 전처리 함수를 작성하겠습니다. 먼저 HTML을 일반 텍스트로 변환하는 함수가 필요합니다. 이 작업에는 당연히 [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) 라이브러리를 사용하는게 좋지만 의존성을 줄이기 위해서 정규식을 사용하여 대강 만들어 보겠습니다([un̨ho͞ly radiańcé destro҉ying all enli̍̈́̂̈́ghtenment](https://stackoverflow.com/a/1732454/38626)의 위험에도 불구하고). 다음 함수는 `<head>` 섹션을 삭제하고 모든 `<a>` 태그를 HYPERLINK 문자로 바꿉니다. 그런 다음 모든 HTML 태그를 제거하고 텍스트만 남깁니다. 보기 편하게 여러개의 개행 문자를 하나로 만들고 (`&gt;`나 `&nbsp;` 같은) html 엔티티를 복원합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from html import unescape\n",
    "\n",
    "def html_to_plain_text(html):\n",
    "    text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<a\\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<.*?>', '', text, flags=re.M | re.S)\n",
    "    text = re.sub(r'(\\s*\\n)+', '\\n', text, flags=re.M | re.S)\n",
    "    return unescape(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "잘 작동하는지 확인해 보겠습니다. 다음은 HTML 스팸입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HTML><HEAD><TITLE></TITLE><META http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\"><STYLE>A:link {TEX-DECORATION: none}A:active {TEXT-DECORATION: none}A:visited {TEXT-DECORATION: none}A:hover {COLOR: #0033ff; TEXT-DECORATION: underline}</STYLE><META content=\"MSHTML 6.00.2713.1100\" name=\"GENERATOR\"></HEAD>\n",
      "<BODY text=\"#000000\" vLink=\"#0033ff\" link=\"#0033ff\" bgColor=\"#CCCC99\"><TABLE borderColor=\"#660000\" cellSpacing=\"0\" cellPadding=\"0\" border=\"0\" width=\"100%\"><TR><TD bgColor=\"#CCCC99\" valign=\"top\" colspan=\"2\" height=\"27\">\n",
      "<font size=\"6\" face=\"Arial, Helvetica, sans-serif\" color=\"#660000\">\n",
      "<b>OTC</b></font></TD></TR><TR><TD height=\"2\" bgcolor=\"#6a694f\">\n",
      "<font size=\"5\" face=\"Times New Roman, Times, serif\" color=\"#FFFFFF\">\n",
      "<b>&nbsp;Newsletter</b></font></TD><TD height=\"2\" bgcolor=\"#6a694f\"><div align=\"right\"><font color=\"#FFFFFF\">\n",
      "<b>Discover Tomorrow's Winners&nbsp;</b></font></div></TD></TR><TR><TD height=\"25\" colspan=\"2\" bgcolor=\"#CCCC99\"><table width=\"100%\" border=\"0\"  ...\n"
     ]
    }
   ],
   "source": [
    "html_spam_emails = [email for email in X_train[y_train==1]\n",
    "                    if get_email_structure(email) == \"text/html\"]\n",
    "sample_html_spam = html_spam_emails[7]\n",
    "print(sample_html_spam.get_content().strip()[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "변환된 텍스트입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Watch for analyst \"Strong Buy Recommendations\" and several advisory newsletters picking CBYI.  CBYI has filed to be traded on the OTCBB, share prices historically INCREASE when companies get listed on this larger trading exchange. CBYI is trading around 25 cents and should skyrocket to $2.66 - $3.25 a share in the near future.\n",
      "Put CBYI on your watch list, acquire a position TODAY.\n",
      "REASONS TO INVEST IN CBYI\n",
      "A profitable company and is on track to beat ALL earnings estimates!\n",
      "One of the FASTEST growing distributors in environmental & safety equipment instruments.\n",
      "Excellent management team, several EXCLUSIVE contracts.  IMPRESSIVE client list including the U.S. Air Force, Anheuser-Busch, Chevron Refining and Mitsubishi Heavy Industries, GE-Energy & Environmental Research.\n",
      "RAPIDLY GROWING INDUSTRY\n",
      "Industry revenues exceed $900 million, estimates indicate that there could be as much as $25 billi ...\n"
     ]
    }
   ],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content())[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "아주 좋습니다! 이제 포맷에 상관없이 이메일을 입력으로 받아서 일반 텍스트를 출력하는 함수를 만들겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\", \"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except: # in case of encoding issues\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\":\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "OTC\n",
      " Newsletter\n",
      "Discover Tomorrow's Winners \n",
      "For Immediate Release\n",
      "Cal-Bay (Stock Symbol: CBYI)\n",
      "Wat ...\n"
     ]
    }
   ],
   "source": [
    "print(email_to_text(sample_html_spam)[:100], \"...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "어간 추출을 해보죠! 이 작업을 하려면 자연어 처리 툴킷([NLTK](http://www.nltk.org/))을 설치해야 합니다. 다음 명령으로 간단히 설치할 수 있습니다(먼저 virtualenv 환경을 활성화시켜야 합니다. 별도의 환경이 없다면 어드민 권한이 필요할지 모릅니다. 아니면 `--user` 옵션을 사용하세요):\n",
    "\n",
    "`$ pip install nltk`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computations => comput\n",
      "Computation => comput\n",
      "Computing => comput\n",
      "Computed => comput\n",
      "Compute => comput\n",
      "Compulsive => compuls\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    import nltk\n",
    "\n",
    "    stemmer = nltk.PorterStemmer()\n",
    "    for word in (\"Computations\", \"Computation\", \"Computing\", \"Computed\", \"Compute\", \"Compulsive\"):\n",
    "        print(word, \"=>\", stemmer.stem(word))\n",
    "except ImportError:\n",
    "    print(\"Error: stemming requires the NLTK module.\")\n",
    "    stemmer = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "인터넷 주소는 \"URL\" 문자로 바꾸겠습니다. [정규식](https://mathiasbynens.be/demo/url-regex)을 하드 코딩할 수도 있지만 [urlextract](https://github.com/lipoja/URLExtract) 라이브러리를 사용하겠습니다. 다음 명령으로 설치합니다(먼저 virtualenv 환경을 활성화시켜야 합니다. 별도의 환경이 없다면 어드민 권한이 필요할지 모릅니다. 아니면 `--user` 옵션을 사용하세요):\n",
    "\n",
    "`$ pip install urlextract`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['github.com', 'https://youtu.be/7Pq-S557XQU?t=3m32s']\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    import urlextract # 루트 도메인 이름을 다운로드하기 위해 인터넷 연결이 필요할지 모릅니다\n",
    "    \n",
    "    url_extractor = urlextract.URLExtract()\n",
    "    print(url_extractor.find_urls(\"Will it detect github.com and https://youtu.be/7Pq-S557XQU?t=3m32s\"))\n",
    "except ImportError:\n",
    "    print(\"Error: replacing URLs requires the urlextract module.\")\n",
    "    url_extractor = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이들을 모두 하나의 변환기로 연결하여 이메일을 단어 카운트로 바꿀 것입니다. 파이썬의 `split()` 메서드를 사용하면 구둣점과 단어 경계를 기준으로 문장을 단어로 바꿉니다. 이 방법이 많은 언어에 통하지만 전부는 아닙니다. 예를 들어 중국어와 일본어는 일반적으로 단어 사이에 공백을 두지 않습니다. 베트남어는 음절 사이에 공백을 두기도 합니다. 여기서는 데이터셋이 (거의) 영어로 되어 있기 때문에 문제없습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, strip_headers=True, lower_case=True, remove_punctuation=True,\n",
    "                 replace_urls=True, replace_numbers=True, stemming=True):\n",
    "        self.strip_headers = strip_headers\n",
    "        self.lower_case = lower_case\n",
    "        self.remove_punctuation = remove_punctuation\n",
    "        self.replace_urls = replace_urls\n",
    "        self.replace_numbers = replace_numbers\n",
    "        self.stemming = stemming\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X_transformed = []\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"\n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            if self.replace_urls and url_extractor is not None:\n",
    "                urls = list(set(url_extractor.find_urls(text)))\n",
    "                urls.sort(key=lambda url: len(url), reverse=True)\n",
    "                for url in urls:\n",
    "                    text = text.replace(url, \" URL \")\n",
    "            if self.replace_numbers:\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?', 'NUMBER', text)\n",
    "            if self.remove_punctuation:\n",
    "                text = re.sub(r'\\W+', ' ', text, flags=re.M)\n",
    "            word_counts = Counter(text.split())\n",
    "            if self.stemming and stemmer is not None:\n",
    "                stemmed_word_counts = Counter()\n",
    "                for word, count in word_counts.items():\n",
    "                    stemmed_word = stemmer.stem(word)\n",
    "                    stemmed_word_counts[stemmed_word] += count\n",
    "                word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 변환기를 몇 개의 이메일에 적용해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([Counter({'r': 1, 'yawn': 1, 'wrote': 1, 'stuff': 1, 'chuck': 1, 'murcko': 1}),\n",
       "       Counter({'the': 11, 'of': 9, 'and': 8, 'christian': 3, 'to': 3, 'all': 3, 'by': 3, 'jesu': 2, 'i': 2, 'been': 2, 'on': 2, 'rogueri': 2, 'half': 2, 'jefferson': 2, 'one': 2, 'teach': 2, 'ha': 2, 'superstit': 2, 'have': 2, 'thoma': 1, 'featur': 1, 'world': 1, 'upon': 1, 'support': 1, 'do': 1, 'burnt': 1, 'other': 1, 'e': 1, 'redeem': 1, 'known': 1, 'fine': 1, 'introduct': 1, 'a': 1, 'tortur': 1, 'sinc': 1, 'ever': 1, 'that': 1, 'most': 1, 'great': 1, 'over': 1, 'system': 1, 'william': 1, 'women': 1, 'again': 1, 'first': 1, 'letter': 1, 'absurd': 1, 'what': 1, 'hypocrit': 1, 'john': 1, 'are': 1, 'million': 1, 'mytholog': 1, 'found': 1, 'dupe': 1, 'in': 1, 'alik': 1, 'interest': 1, 'url': 1, 'remsburg': 1, 'man': 1, 'paul': 1, 'word': 1, 'our': 1, 'pervert': 1, 'not': 1, 'innoc': 1, 'quot': 1, 'six': 1, 'band': 1, 'becom': 1, 'fool': 1, 'they': 1, 'coercion': 1, 'examin': 1, 'men': 1, 'effect': 1, 'children': 1, 'untruth': 1, 'were': 1, 'imprison': 1, 'thi': 1, 'fabl': 1, 'short': 1, 'error': 1, 'shone': 1, 'make': 1, 'american': 1, 'larg': 1, 'some': 1, 'perpetr': 1, 'particular': 1, 'corrupt': 1, 'import': 1, 'histor': 1, 'earth': 1, 'led': 1, 'find': 1}),\n",
       "       Counter({'url': 5, 's': 3, 'group': 3, 'to': 3, 'forteana': 2, 'martin': 2, 'unsubscrib': 2, 'and': 2, 'we': 2, 'is': 2, 'an': 2, 'in': 2, 'yahoo': 2, 'hamza': 1, 'for': 1, 'non': 1, 'rob': 1, 'hi': 1, 'send': 1, 'includ': 1, 'career': 1, 'use': 1, 'from': 1, 'all': 1, 'memri': 1, 'subject': 1, 'more': 1, 't': 1, 'altern': 1, 'html': 1, 'know': 1, 'sponsor': 1, 'wrote': 1, 'should': 1, 'belief': 1, 'yemen': 1, 'y': 1, 'base': 1, 'that': 1, 'thi': 1, 'adamson': 1, 'outright': 1, 'your': 1, 'dvd': 1, 'rundown': 1, 'on': 1, 'factual': 1, 'how': 1, 'number': 1, 'don': 1, 'unbias': 1, 'muslim': 1, 'email': 1, 'rather': 1, 'be': 1, 'free': 1, 'now': 1, 'of': 1, 'p': 1, 'murder': 1, 'join': 1})],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few = X_train[:3]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "제대로 작동하는 것 같네요!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 단어 카운트를 벡터로 변환해야 합니다. 이를 위해서 또 다른 변환기를 만들겠습니다. 이 변환기는 (자주 나타나는 단어 순으로 정렬된) 어휘 목록을 구축하는 `fit()` 메서드와 어휘 목록을 사용해 단어를 벡터로 바꾸는 `transform()` 메서드를 가집니다. 출력은 희소 행렬이 됩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, vocabulary_size=1000):\n",
    "        self.vocabulary_size = vocabulary_size\n",
    "    def fit(self, X, y=None):\n",
    "        total_count = Counter()\n",
    "        for word_count in X:\n",
    "            for word, count in word_count.items():\n",
    "                total_count[word] += min(count, 10)\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row, word_count in enumerate(X):\n",
    "            for word, count in word_count.items():\n",
    "                rows.append(row)\n",
    "                cols.append(self.vocabulary_.get(word, 0))\n",
    "                data.append(count)\n",
    "        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3x11 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 20 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=10)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 6,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [99, 11,  9,  8,  1,  3,  3,  3,  2,  1,  3],\n",
       "       [65,  0,  1,  2,  5,  3,  1,  0,  1,  2,  0]], dtype=int64)"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few_vectors.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 행렬은 무엇을 의미하나요? 세 번째 행의 첫 번째 열의 92는 세 번째 이메일이 어휘 목록에 없는 단어를 92개 가지고 있다는 뜻입니다. 그 다음의 1은 어휘 목록에 있는 첫 번째 단어가 한 번 등장한다는 뜻이고 그 다음의 0은 한 번도 나타나지 않는다는 뜻입니다. 이 단어들이 무엇인지 확인하려면 어휘 목록을 보면 됩니다. 첫 번째 단어는 \"the\"이고 두 번째 단어는 \"of\"입니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'all': 6,\n",
       " 'and': 3,\n",
       " 'by': 10,\n",
       " 'christian': 7,\n",
       " 'in': 9,\n",
       " 'of': 2,\n",
       " 'on': 8,\n",
       " 'the': 1,\n",
       " 'to': 5,\n",
       " 'url': 4}"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer.vocabulary_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 스팸 분류기를 훈련시킬 준비를 마쳤습니다! 전체 데이터셋을 변환시켜보죠:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_to_wordcount\", EmailToWordCounterTransformer()),\n",
    "    (\"wordcount_to_vector\", WordCounterToVectorTransformer()),\n",
    "])\n",
    "\n",
    "X_train_transformed = preprocess_pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  ................................................................\n",
      "[CV] .................................... , score=0.985, total=   0.0s\n",
      "[CV]  ................................................................\n",
      "[CV] .................................... , score=0.985, total=   0.0s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................... , score=0.9925, total=   0.1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.2s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9874999999999999"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "log_clf = LogisticRegression()\n",
    "score = cross_val_score(log_clf, X_train_transformed, y_train, cv=3, verbose=3)\n",
    "score.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "98.7%가 넘네요. 첫 번째 시도치고 나쁘지 않습니다! :) 그러나 이 데이터셋은 비교적 쉬운 문제입니다. 더 어려운 데이터셋에 적용해 보면 결과가 그리 높지 않을 것입니다. 여러개의 모델을 시도해 보고 제일 좋은 것을 골라 교차 검증으로 세밀하게 튜닝해 보세요.\n",
    "\n",
    "하지만 전체 내용을 파악했으므로 여기서 멈추겠습니다. 테스트 세트에서 정밀도/재현율을 출력해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 94.90%\n",
      "Recall: 97.89%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)\n",
    "\n",
    "log_clf = LogisticRegression()\n",
    "log_clf.fit(X_train_transformed, y_train)\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)\n",
    "\n",
    "print(\"정밀도: {:.2f}%\".format(100 * precision_score(y_test, y_pred)))\n",
    "print(\"재현율: {:.2f}%\".format(100 * recall_score(y_test, y_pred)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
