{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.5.5\n",
      "IPython 6.2.1\n",
      "\n",
      "numpy 1.14.1\n",
      "scipy 1.0.0\n",
      "sklearn 0.19.1\n",
      "pandas 0.22.0\n",
      "matplotlib 2.2.2\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,scipy,sklearn,pandas,matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**7장 – 앙상블 학습과 랜덤 포레스트**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 7장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._\n",
    "\n",
    "-여기에서 발생하는 경고는 NumPy에 관련된 것([#10449](https://github.com/scikit-learn/scikit-learn/issues/10449))으로 향후 버전에서는 사라질 것입니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "np.random.seed(42)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ensembles\"\n",
    "\n",
    "def image_path(fig_id):\n",
    "    return os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(image_path(fig_id) + \".png\", format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 투표기반 분류기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "heads_proba = 0.51\n",
    "coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)\n",
    "cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 51000 characters]\n",
      "text/plain": [
       "<Figure size 576x252 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,3.5))\n",
    "plt.plot(cumulative_heads_ratio)\n",
    "plt.plot([0, 10000], [0.51, 0.51], \"k--\", linewidth=2, label=\"51%\")\n",
    "plt.plot([0, 10000], [0.5, 0.5], \"k-\", label=\"50%\")\n",
    "plt.xlabel(\"동전을 던진 횟수\")\n",
    "plt.ylabel(\"앞면이 나온 비율\")\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.axis([0, 10000, 0.42, 0.58])\n",
    "save_fig(\"law_of_large_numbers_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('lr', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=42, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)), ('rf', RandomFor...f',\n",
       "  max_iter=-1, probability=False, random_state=42, shrinking=True,\n",
       "  tol=0.001, verbose=False))],\n",
       "         flatten_transform=None, n_jobs=1, voting='hard', weights=None)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "log_clf = LogisticRegression(random_state=42)\n",
    "rnd_clf = RandomForestClassifier(random_state=42)\n",
    "svm_clf = SVC(random_state=42)\n",
    "\n",
    "voting_clf = VotingClassifier(\n",
    "    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
    "    voting='hard')\n",
    "voting_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression 0.864\n",
      "RandomForestClassifier 0.872\n",
      "SVC 0.888\n",
      "VotingClassifier 0.896\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/haesun/anaconda3/envs/handson-ml/lib/python3.5/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('lr', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=42, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)), ('rf', RandomFor...bf',\n",
       "  max_iter=-1, probability=True, random_state=42, shrinking=True,\n",
       "  tol=0.001, verbose=False))],\n",
       "         flatten_transform=None, n_jobs=1, voting='soft', weights=None)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_clf = LogisticRegression(random_state=42)\n",
    "rnd_clf = RandomForestClassifier(random_state=42)\n",
    "svm_clf = SVC(probability=True, random_state=42)\n",
    "\n",
    "voting_clf = VotingClassifier(\n",
    "    estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
    "    voting='soft')\n",
    "voting_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression 0.864\n",
      "RandomForestClassifier 0.872\n",
      "SVC 0.888\n",
      "VotingClassifier 0.912\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/haesun/anaconda3/envs/handson-ml/lib/python3.5/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 배깅 앙상블"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "bag_clf = BaggingClassifier(\n",
    "    DecisionTreeClassifier(random_state=42), n_estimators=500,\n",
    "    max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)\n",
    "bag_clf.fit(X_train, y_train)\n",
    "y_pred = bag_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.904\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.856\n"
     ]
    }
   ],
   "source": [
    "tree_clf = DecisionTreeClassifier(random_state=42)\n",
    "tree_clf.fit(X_train, y_train)\n",
    "y_pred_tree = tree_clf.predict(X_test)\n",
    "print(accuracy_score(y_test, y_pred_tree))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n",
    "    x1s = np.linspace(axes[0], axes[1], 100)\n",
    "    x2s = np.linspace(axes[2], axes[3], 100)\n",
    "    x1, x2 = np.meshgrid(x1s, x2s)\n",
    "    X_new = np.c_[x1.ravel(), x2.ravel()]\n",
    "    y_pred = clf.predict(X_new).reshape(x1.shape)\n",
    "    custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n",
    "    plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)\n",
    "    if contour:\n",
    "        custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n",
    "        plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n",
    "    plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n",
    "    plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n",
    "    plt.axis(axes)\n",
    "    plt.xlabel(r\"$x_1$\", fontsize=18)\n",
    "    plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 66372 characters]\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "plt.subplot(121)\n",
    "plot_decision_boundary(tree_clf, X, y)\n",
    "plt.title(\"결정 트리\", fontsize=14)\n",
    "plt.subplot(122)\n",
    "plot_decision_boundary(bag_clf, X, y)\n",
    "plt.title(\"배깅을 사용한 결정 트리\", fontsize=14)\n",
    "save_fig(\"decision_tree_without_and_with_bagging_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 랜덤 포레스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_clf = BaggingClassifier(\n",
    "    DecisionTreeClassifier(splitter=\"random\", max_leaf_nodes=16, random_state=42),\n",
    "    n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_clf.fit(X_train, y_train)\n",
    "y_pred = bag_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)\n",
    "rnd_clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred_rf = rnd_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.976"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(y_pred == y_pred_rf) / len(y_pred)  # 거의 동일한 예측"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sepal length (cm) 0.11249225099876374\n",
      "sepal width (cm) 0.023119288282510326\n",
      "petal length (cm) 0.44103046436395765\n",
      "petal width (cm) 0.4233579963547681\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "iris = load_iris()\n",
    "rnd_clf = RandomForestClassifier(n_estimators=500, n_jobs=-1, random_state=42)\n",
    "rnd_clf.fit(iris[\"data\"], iris[\"target\"])\n",
    "for name, score in zip(iris[\"feature_names\"], rnd_clf.feature_importances_):\n",
    "    print(name, score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.11249225, 0.02311929, 0.44103046, 0.423358  ])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_clf.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 46944 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "for i in range(15):\n",
    "    tree_clf = DecisionTreeClassifier(max_leaf_nodes=16, random_state=42 + i)\n",
    "    indices_with_replacement = np.random.randint(0, len(X_train), len(X_train))\n",
    "    tree_clf.fit(X[indices_with_replacement], y[indices_with_replacement])\n",
    "    plot_decision_boundary(tree_clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.02, contour=False)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## oob 평가"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9013333333333333"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_clf = BaggingClassifier(\n",
    "    DecisionTreeClassifier(random_state=42), n_estimators=500,\n",
    "    bootstrap=True, n_jobs=-1, oob_score=True, random_state=40)\n",
    "bag_clf.fit(X_train, y_train)\n",
    "bag_clf.oob_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.31746032, 0.68253968],\n",
       "       [0.34117647, 0.65882353],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.08379888, 0.91620112],\n",
       "       [0.31693989, 0.68306011],\n",
       "       [0.02923977, 0.97076023],\n",
       "       [0.97687861, 0.02312139],\n",
       "       [0.97765363, 0.02234637],\n",
       "       [0.74404762, 0.25595238],\n",
       "       [0.        , 1.        ],\n",
       "       [0.71195652, 0.28804348],\n",
       "       [0.83957219, 0.16042781],\n",
       "       [0.97777778, 0.02222222],\n",
       "       [0.0625    , 0.9375    ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.97297297, 0.02702703],\n",
       "       [0.95238095, 0.04761905],\n",
       "       [1.        , 0.        ],\n",
       "       [0.01704545, 0.98295455],\n",
       "       [0.38947368, 0.61052632],\n",
       "       [0.88700565, 0.11299435],\n",
       "       [1.        , 0.        ],\n",
       "       [0.96685083, 0.03314917],\n",
       "       [0.        , 1.        ],\n",
       "       [0.99428571, 0.00571429],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.64804469, 0.35195531],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.13402062, 0.86597938],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.36065574, 0.63934426],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.27093596, 0.72906404],\n",
       "       [0.34146341, 0.65853659],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.00531915, 0.99468085],\n",
       "       [0.98265896, 0.01734104],\n",
       "       [0.91428571, 0.08571429],\n",
       "       [0.97282609, 0.02717391],\n",
       "       [0.97029703, 0.02970297],\n",
       "       [0.        , 1.        ],\n",
       "       [0.06134969, 0.93865031],\n",
       "       [0.98019802, 0.01980198],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.97790055, 0.02209945],\n",
       "       [0.79473684, 0.20526316],\n",
       "       [0.41919192, 0.58080808],\n",
       "       [0.99473684, 0.00526316],\n",
       "       [0.        , 1.        ],\n",
       "       [0.67613636, 0.32386364],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.87356322, 0.12643678],\n",
       "       [1.        , 0.        ],\n",
       "       [0.56140351, 0.43859649],\n",
       "       [0.16304348, 0.83695652],\n",
       "       [0.67539267, 0.32460733],\n",
       "       [0.90673575, 0.09326425],\n",
       "       [0.        , 1.        ],\n",
       "       [0.16201117, 0.83798883],\n",
       "       [0.89005236, 0.10994764],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.995     , 0.005     ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.07272727, 0.92727273],\n",
       "       [0.05418719, 0.94581281],\n",
       "       [0.29533679, 0.70466321],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.81871345, 0.18128655],\n",
       "       [0.01092896, 0.98907104],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.22513089, 0.77486911],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.9368932 , 0.0631068 ],\n",
       "       [0.76536313, 0.23463687],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.17127072, 0.82872928],\n",
       "       [0.65306122, 0.34693878],\n",
       "       [0.        , 1.        ],\n",
       "       [0.03076923, 0.96923077],\n",
       "       [0.49444444, 0.50555556],\n",
       "       [1.        , 0.        ],\n",
       "       [0.02673797, 0.97326203],\n",
       "       [0.98870056, 0.01129944],\n",
       "       [0.23121387, 0.76878613],\n",
       "       [0.5       , 0.5       ],\n",
       "       [0.9947644 , 0.0052356 ],\n",
       "       [0.00555556, 0.99444444],\n",
       "       [0.98963731, 0.01036269],\n",
       "       [0.25641026, 0.74358974],\n",
       "       [0.92972973, 0.07027027],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.80681818, 0.19318182],\n",
       "       [1.        , 0.        ],\n",
       "       [0.0106383 , 0.9893617 ],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.98181818, 0.01818182],\n",
       "       [1.        , 0.        ],\n",
       "       [0.01036269, 0.98963731],\n",
       "       [0.97752809, 0.02247191],\n",
       "       [0.99453552, 0.00546448],\n",
       "       [0.01960784, 0.98039216],\n",
       "       [0.18367347, 0.81632653],\n",
       "       [0.98387097, 0.01612903],\n",
       "       [0.29533679, 0.70466321],\n",
       "       [0.98295455, 0.01704545],\n",
       "       [0.        , 1.        ],\n",
       "       [0.00561798, 0.99438202],\n",
       "       [0.75138122, 0.24861878],\n",
       "       [0.38624339, 0.61375661],\n",
       "       [0.42708333, 0.57291667],\n",
       "       [0.86315789, 0.13684211],\n",
       "       [0.92964824, 0.07035176],\n",
       "       [0.05699482, 0.94300518],\n",
       "       [0.82802548, 0.17197452],\n",
       "       [0.01546392, 0.98453608],\n",
       "       [0.        , 1.        ],\n",
       "       [0.02298851, 0.97701149],\n",
       "       [0.96721311, 0.03278689],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.01041667, 0.98958333],\n",
       "       [0.        , 1.        ],\n",
       "       [0.0326087 , 0.9673913 ],\n",
       "       [0.01020408, 0.98979592],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.93785311, 0.06214689],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.99462366, 0.00537634],\n",
       "       [0.        , 1.        ],\n",
       "       [0.38860104, 0.61139896],\n",
       "       [0.32065217, 0.67934783],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.31182796, 0.68817204],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.00588235, 0.99411765],\n",
       "       [0.        , 1.        ],\n",
       "       [0.98387097, 0.01612903],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.62264151, 0.37735849],\n",
       "       [0.92344498, 0.07655502],\n",
       "       [0.        , 1.        ],\n",
       "       [0.99526066, 0.00473934],\n",
       "       [1.        , 0.        ],\n",
       "       [0.98888889, 0.01111111],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.06451613, 0.93548387],\n",
       "       [1.        , 0.        ],\n",
       "       [0.05154639, 0.94845361],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.03278689, 0.96721311],\n",
       "       [1.        , 0.        ],\n",
       "       [0.95808383, 0.04191617],\n",
       "       [0.79532164, 0.20467836],\n",
       "       [0.55665025, 0.44334975],\n",
       "       [0.        , 1.        ],\n",
       "       [0.18604651, 0.81395349],\n",
       "       [1.        , 0.        ],\n",
       "       [0.93121693, 0.06878307],\n",
       "       [0.97740113, 0.02259887],\n",
       "       [1.        , 0.        ],\n",
       "       [0.00531915, 0.99468085],\n",
       "       [0.        , 1.        ],\n",
       "       [0.44623656, 0.55376344],\n",
       "       [0.86363636, 0.13636364],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.00558659, 0.99441341],\n",
       "       [0.        , 1.        ],\n",
       "       [0.96923077, 0.03076923],\n",
       "       [0.        , 1.        ],\n",
       "       [0.21649485, 0.78350515],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.98477157, 0.01522843],\n",
       "       [0.8       , 0.2       ],\n",
       "       [0.99441341, 0.00558659],\n",
       "       [0.        , 1.        ],\n",
       "       [0.08379888, 0.91620112],\n",
       "       [0.98984772, 0.01015228],\n",
       "       [0.01142857, 0.98857143],\n",
       "       [0.        , 1.        ],\n",
       "       [0.02747253, 0.97252747],\n",
       "       [1.        , 0.        ],\n",
       "       [0.79144385, 0.20855615],\n",
       "       [0.        , 1.        ],\n",
       "       [0.90804598, 0.09195402],\n",
       "       [0.98387097, 0.01612903],\n",
       "       [0.20634921, 0.79365079],\n",
       "       [0.19767442, 0.80232558],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.20338983, 0.79661017],\n",
       "       [0.98181818, 0.01818182],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.98969072, 0.01030928],\n",
       "       [0.        , 1.        ],\n",
       "       [0.48663102, 0.51336898],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.07821229, 0.92178771],\n",
       "       [0.11176471, 0.88823529],\n",
       "       [0.99415205, 0.00584795],\n",
       "       [0.03015075, 0.96984925],\n",
       "       [1.        , 0.        ],\n",
       "       [0.40837696, 0.59162304],\n",
       "       [0.04891304, 0.95108696],\n",
       "       [0.51595745, 0.48404255],\n",
       "       [0.51898734, 0.48101266],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.59903382, 0.40096618],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.24157303, 0.75842697],\n",
       "       [0.81052632, 0.18947368],\n",
       "       [0.08717949, 0.91282051],\n",
       "       [0.99453552, 0.00546448],\n",
       "       [0.82142857, 0.17857143],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.125     , 0.875     ],\n",
       "       [0.04712042, 0.95287958],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.89150943, 0.10849057],\n",
       "       [0.1978022 , 0.8021978 ],\n",
       "       [0.95238095, 0.04761905],\n",
       "       [0.00515464, 0.99484536],\n",
       "       [0.609375  , 0.390625  ],\n",
       "       [0.07692308, 0.92307692],\n",
       "       [0.99484536, 0.00515464],\n",
       "       [0.84210526, 0.15789474],\n",
       "       [0.        , 1.        ],\n",
       "       [0.99484536, 0.00515464],\n",
       "       [0.95876289, 0.04123711],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.26903553, 0.73096447],\n",
       "       [0.98461538, 0.01538462],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.00574713, 0.99425287],\n",
       "       [0.85142857, 0.14857143],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.76506024, 0.23493976],\n",
       "       [0.8969697 , 0.1030303 ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.73333333, 0.26666667],\n",
       "       [0.47727273, 0.52272727],\n",
       "       [0.        , 1.        ],\n",
       "       [0.92473118, 0.07526882],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.87709497, 0.12290503],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.74752475, 0.25247525],\n",
       "       [0.09146341, 0.90853659],\n",
       "       [0.44329897, 0.55670103],\n",
       "       [0.22395833, 0.77604167],\n",
       "       [0.        , 1.        ],\n",
       "       [0.87046632, 0.12953368],\n",
       "       [0.78212291, 0.21787709],\n",
       "       [0.00507614, 0.99492386],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.02884615, 0.97115385],\n",
       "       [0.96571429, 0.03428571],\n",
       "       [0.93478261, 0.06521739],\n",
       "       [1.        , 0.        ],\n",
       "       [0.49756098, 0.50243902],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.01604278, 0.98395722],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.96987952, 0.03012048],\n",
       "       [0.        , 1.        ],\n",
       "       [0.05747126, 0.94252874],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.98989899, 0.01010101],\n",
       "       [0.01675978, 0.98324022],\n",
       "       [1.        , 0.        ],\n",
       "       [0.13541667, 0.86458333],\n",
       "       [0.        , 1.        ],\n",
       "       [0.00546448, 0.99453552],\n",
       "       [0.        , 1.        ],\n",
       "       [0.41836735, 0.58163265],\n",
       "       [0.11309524, 0.88690476],\n",
       "       [0.22110553, 0.77889447],\n",
       "       [1.        , 0.        ],\n",
       "       [0.97647059, 0.02352941],\n",
       "       [0.22826087, 0.77173913],\n",
       "       [0.98882682, 0.01117318],\n",
       "       [0.        , 1.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [1.        , 0.        ],\n",
       "       [0.96428571, 0.03571429],\n",
       "       [0.33507853, 0.66492147],\n",
       "       [0.98235294, 0.01764706],\n",
       "       [1.        , 0.        ],\n",
       "       [0.        , 1.        ],\n",
       "       [0.99465241, 0.00534759],\n",
       "       [0.        , 1.        ],\n",
       "       [0.06043956, 0.93956044],\n",
       "       [0.97619048, 0.02380952],\n",
       "       [1.        , 0.        ],\n",
       "       [0.03108808, 0.96891192],\n",
       "       [0.57291667, 0.42708333]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_clf.oob_decision_function_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.912"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "y_pred = bag_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 특성 중요도"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "mnist = fetch_mldata('MNIST original')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_clf = RandomForestClassifier(random_state=42)\n",
    "rnd_clf.fit(mnist[\"data\"], mnist[\"target\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_digit(data):\n",
    "    image = data.reshape(28, 28)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.hot,\n",
    "               interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 7944 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_digit(rnd_clf.feature_importances_)\n",
    "\n",
    "cbar = plt.colorbar(ticks=[rnd_clf.feature_importances_.min(), rnd_clf.feature_importances_.max()])\n",
    "cbar.ax.set_yticklabels(['중요하지 않음', '매우 중요함'])\n",
    "\n",
    "save_fig(\"mnist_feature_importance_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 아다부스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "          base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=1,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "          learning_rate=0.5, n_estimators=200, random_state=42)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "ada_clf = AdaBoostClassifier(\n",
    "    DecisionTreeClassifier(max_depth=1), n_estimators=200,\n",
    "    algorithm=\"SAMME.R\", learning_rate=0.5, random_state=42)\n",
    "ada_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37584 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_decision_boundary(ada_clf, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 117112 characters]\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "m = len(X_train)\n",
    "\n",
    "plt.figure(figsize=(11, 4))\n",
    "for subplot, learning_rate in ((121, 1), (122, 0.5)):\n",
    "    sample_weights = np.ones(m)\n",
    "    plt.subplot(subplot)\n",
    "    if subplot == 121:\n",
    "        plt.text(-0.7, -0.65, \"1\", fontsize=14)\n",
    "        plt.text(-0.6, -0.10, \"2\", fontsize=14)\n",
    "        plt.text(-0.5,  0.10, \"3\", fontsize=14)\n",
    "        plt.text(-0.4,  0.55, \"4\", fontsize=14)\n",
    "        plt.text(-0.3,  0.90, \"5\", fontsize=14)        \n",
    "    for i in range(5):\n",
    "        svm_clf = SVC(kernel=\"rbf\", C=0.05, random_state=42)\n",
    "        svm_clf.fit(X_train, y_train, sample_weight=sample_weights)\n",
    "        y_pred = svm_clf.predict(X_train)\n",
    "        sample_weights[y_pred != y_train] *= (1 + learning_rate)\n",
    "        plot_decision_boundary(svm_clf, X, y, alpha=0.2)\n",
    "        plt.title(\"learning_rate = {}\".format(learning_rate), fontsize=16)\n",
    "\n",
    "save_fig(\"boosting_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['base_estimator_',\n",
       " 'classes_',\n",
       " 'estimator_errors_',\n",
       " 'estimator_weights_',\n",
       " 'estimators_',\n",
       " 'feature_importances_',\n",
       " 'n_classes_']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(m for m in dir(ada_clf) if not m.startswith(\"_\") and m.endswith(\"_\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 그래디언트 부스팅"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "X = np.random.rand(100, 1) - 0.5\n",
    "y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,\n",
       "           max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           presort=False, random_state=42, splitter='best')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
    "tree_reg1.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,\n",
       "           max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           presort=False, random_state=42, splitter='best')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y2 = y - tree_reg1.predict(X)\n",
    "tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
    "tree_reg2.fit(X, y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=2, max_features=None,\n",
       "           max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           presort=False, random_state=42, splitter='best')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y3 = y2 - tree_reg2.predict(X)\n",
    "tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)\n",
    "tree_reg3.fit(X, y3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new = np.array([[0.8]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = sum(tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.75026781])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 75088 characters]\n",
      "text/plain": [
       "<Figure size 792x792 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_predictions(regressors, X, y, axes, label=None, style=\"r-\", data_style=\"b.\", data_label=None):\n",
    "    x1 = np.linspace(axes[0], axes[1], 500)\n",
    "    y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)\n",
    "    plt.plot(X[:, 0], y, data_style, label=data_label)\n",
    "    plt.plot(x1, y_pred, style, linewidth=2, label=label)\n",
    "    if label or data_label:\n",
    "        plt.legend(loc=\"upper center\", fontsize=16)\n",
    "    plt.axis(axes)\n",
    "\n",
    "plt.figure(figsize=(11,11))\n",
    "\n",
    "plt.subplot(321)\n",
    "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h_1(x_1)$\", style=\"g-\", data_label=\"훈련 세트\")\n",
    "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
    "plt.title(\"잔여 오차와 트리의 예측\", fontsize=16)\n",
    "\n",
    "plt.subplot(322)\n",
    "plot_predictions([tree_reg1], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1)$\", data_label=\"훈련 세트\")\n",
    "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
    "plt.title(\"앙상블의 예측\", fontsize=16)\n",
    "\n",
    "plt.subplot(323)\n",
    "plot_predictions([tree_reg2], X, y2, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_2(x_1)$\", style=\"g-\", data_style=\"k+\", data_label=\"잔여 오차\")\n",
    "plt.ylabel(\"$y - h_1(x_1)$\", fontsize=16)\n",
    "\n",
    "plt.subplot(324)\n",
    "plot_predictions([tree_reg1, tree_reg2], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1)$\")\n",
    "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
    "\n",
    "plt.subplot(325)\n",
    "plot_predictions([tree_reg3], X, y3, axes=[-0.5, 0.5, -0.5, 0.5], label=\"$h_3(x_1)$\", style=\"g-\", data_style=\"k+\")\n",
    "plt.ylabel(\"$y - h_1(x_1) - h_2(x_1)$\", fontsize=16)\n",
    "plt.xlabel(\"$x_1$\", fontsize=16)\n",
    "\n",
    "plt.subplot(326)\n",
    "plot_predictions([tree_reg1, tree_reg2, tree_reg3], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$\")\n",
    "plt.xlabel(\"$x_1$\", fontsize=16)\n",
    "plt.ylabel(\"$y$\", fontsize=16, rotation=0)\n",
    "\n",
    "save_fig(\"gradient_boosting_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "             learning_rate=0.1, loss='ls', max_depth=2, max_features=None,\n",
       "             max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "             min_impurity_split=None, min_samples_leaf=1,\n",
       "             min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "             n_estimators=3, presort='auto', random_state=42,\n",
       "             subsample=1.0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=0.1, random_state=42)\n",
    "gbrt.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "             learning_rate=0.1, loss='ls', max_depth=2, max_features=None,\n",
       "             max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "             min_impurity_split=None, min_samples_leaf=1,\n",
       "             min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "             n_estimators=200, presort='auto', random_state=42,\n",
       "             subsample=1.0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)\n",
    "gbrt_slow.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 30400 characters]\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"앙상블의 예측\")\n",
    "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)\n",
    "\n",
    "plt.subplot(122)\n",
    "plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n",
    "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)\n",
    "\n",
    "save_fig(\"gbrt_learning_rate_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 조기 종료를 사용한 그래디언트 부스팅"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "             learning_rate=0.1, loss='ls', max_depth=2, max_features=None,\n",
       "             max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "             min_impurity_split=None, min_samples_leaf=1,\n",
       "             min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "             n_estimators=55, presort='auto', random_state=42,\n",
       "             subsample=1.0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=49)\n",
    "\n",
    "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120, random_state=42)\n",
    "gbrt.fit(X_train, y_train)\n",
    "\n",
    "errors = [mean_squared_error(y_val, y_pred)\n",
    "          for y_pred in gbrt.staged_predict(X_val)]\n",
    "bst_n_estimators = np.argmin(errors)\n",
    "\n",
    "gbrt_best = GradientBoostingRegressor(max_depth=2,n_estimators=bst_n_estimators, random_state=42)\n",
    "gbrt_best.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_error = np.min(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 28420 characters]\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11, 4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(errors, \"b.-\")\n",
    "plt.plot([bst_n_estimators, bst_n_estimators], [0, min_error], \"k--\")\n",
    "plt.plot([0, 120], [min_error, min_error], \"k--\")\n",
    "plt.plot(bst_n_estimators, min_error, \"ko\")\n",
    "plt.text(bst_n_estimators, min_error*1.2, \"최소\", ha=\"center\", fontsize=14)\n",
    "plt.axis([0, 120, 0, 0.01])\n",
    "plt.xlabel(\"트리 개수\")\n",
    "plt.title(\"검증 오차\", fontsize=14)\n",
    "\n",
    "plt.subplot(122)\n",
    "plot_predictions([gbrt_best], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n",
    "plt.title(\"최적 모델 (트리 %d개)\" % bst_n_estimators, fontsize=14)\n",
    "\n",
    "save_fig(\"early_stopping_gbrt_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingRegressor(max_depth=2, warm_start=True, random_state=42)\n",
    "\n",
    "min_val_error = float(\"inf\")\n",
    "error_going_up = 0\n",
    "for n_estimators in range(1, 120):\n",
    "    gbrt.n_estimators = n_estimators\n",
    "    gbrt.fit(X_train, y_train)\n",
    "    y_pred = gbrt.predict(X_val)\n",
    "    val_error = mean_squared_error(y_val, y_pred)\n",
    "    if val_error < min_val_error:\n",
    "        min_val_error = val_error\n",
    "        error_going_up = 0\n",
    "    else:\n",
    "        error_going_up += 1\n",
    "        if error_going_up == 5:\n",
    "            break  # early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "61\n"
     ]
    }
   ],
   "source": [
    "print(gbrt.n_estimators)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 해답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**곧 제공됩니다**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "nav_menu": {
   "height": "252px",
   "width": "333px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
