{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.6.5\n",
      "IPython 6.4.0\n",
      "\n",
      "numpy 1.14.3\n",
      "scipy 1.1.0\n",
      "matplotlib 2.2.2\n",
      "tensorflow 1.8.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,scipy,matplotlib,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**10장 – 인공 신경망 소개**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 10장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 퍼셉트론"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # 꽃잎 길이, 꽃잎 너비\n",
    "y = (iris.target == 0).astype(np.int)\n",
    "\n",
    "per_clf = Perceptron(max_iter=100, random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAEYCAYAAABBfQDEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xt8zuUfx/HXtYM5zpkcI4fScl5IBzpISaWzQiLkEP2KVCpKm8MwItJRhaKDDqISOR/GpMg5hZBz5rjZdl+/Pza3mZ1t9/fe9n4+Ht+H3Z/v6XOP2WfXru/1MdZaREREREQkgY/TCYiIiIiIeBMVyCIiIiIiSahAFhERERFJQgWyiIiIiEgSKpBFRERERJJQgSwiIiIikoQKZBERERGRJDxaIBtjnjLG/G6MiTTGrDfG9E7luD7GmC3GmD+MMd8bY8pnZJ+IiIiIyKXyWIFsjPEFagHXW2uDgTuBMcaYSsmOawm8ANxorb0GiAQ+SG+fiIiIiEh2ME510jPGFAQOAHWttbuTxN8G/rPWDkp8XRI4BJQGRqS2z1ob5eG3ICIiIiJ5kJ+D9x4HzExaHCe6Avj83Atr7X/GmCigWjr7fk96EWNMD6AHQJEiBRtfeeUFA9VeLTY2jr//PsDJk9HumI+PL6VLV6dQoeIOZiYiIiLinN3Jq8YkqlbNyDk7sfawSe8+jhTIxpgQoBLwQEq7gfhksTgSpoOkte8C1tp3gXcBGjeuaVetGnOJWXtWfHw8w4d/SUjITFwuFy5XPIcO/clttz1Hu3bD8fMr4HSKIiIiIh7Vs2fq+wYNysg5wRm6j8dXsTDGjAaCgAestWdTOGQPUDXJ8YVJmF6xJ519eYqvry+vvPII8+YNpWLFUu74/PnhjBp1A4cO/eVgdiIiIiJ5lycf0vMxxkwGqgAPnSuOjTG+xpgFxpgbEw+dCnQwxpybS9AHWG6tPZTOvjzpppuuITJyHG3anP+JZ9euNYwaVZfIyJkOZiYiIiLiWYGBmYunty81HntIzxjTFphNwsoTSadJhADjgWettd8mHtsP6ArEAvuAntbaf9Pbl5rcOMUiOWstb775HS+/PJXY2Dh3/IYbuvPww+MoUKCwg9mJiIiIeL+ePc3axNXU0uTYKhaelBcK5HMiI7fTseNo/vrrgDtWsWIQ3bp9TsWKVzuYmYiIiIh3y2iBrE56uUxwcC0iIsJ56KEb3LF9+zYyfHgwy5d/QH74gUdEREQkJ6lAzoWKFy/CtGn9mTy5D4UKJaxmERt7hqlTu/HBB49x5sxxhzMUERERyb00xSKX27hxN489NorNm/9xx8qWrUG3bjO4/PKMLWUiIiIiklcNHAjH3WOHwVgbme46yBpBzuWCgqqycuVounZt5Y4dOrSDsLDmLFgwTlMuREREJF87noVfrKtAzgMKFw5g8uQ+TJ3an2LFCgEQHx/LF188y6RJ93Dy5BGHMxQRERHJPVQg5yGPPHIjERHhNGpUwx3bsOF7QkLqs337UgczExEREck9VCDnMTVrVmDJkhE888w97tixY3sJD2/JnDlv4HIl79QtIiIiIkmpQM6DChTwZ9Sornz99cuULl0MAGtdzJ49mDffbMWxY/sczlBERETEe6lAzsPuuuta1qwZy403BrljW7cuJDS0ARs3/uhgZiIiIiKe4dWtpp2Ul5d5y4i4uHhCQz9n2LDPL1jVolWr52nXLhRfX38HsxMRERHxDHXSEzc/P1+GDHmUn34aSoUKJd3xn38exejRN3L48N8OZiciIiLiXVQg5yMtW9ZlzZqxtG7dyB37++8IQkMbsnbtlw5mJiIiIuI9VCDnM+XKleDbb19hxIgn8PPzBeDMmSjee+8hPv20F2fPnnE4QxERERFn+TmdgHiej48Pzz3XjhtuuJqOHUezc+dBAJYsmcyOHSvo1m0GFSrUcThLERERyct69kx93+TJKcd79YKUHp8zBt5+O3vyAo0g52tNmtRm9epwHniguTu2d+96hg8PZsWKKWpTLSIiIl4ltdIku0sWFcj5XIkSRfn00+eZOLEXBQsWAODs2dN88klXpkzpRHT0CYczFBEREfEsFciCMYbu3VuzfHkYV15Z2R1fvXo6w4Y1ZvfudQ5mJyIiIuJZKpDFrW7daqxaNZrOnW91xw4e3E5YWDMWLpygKRciIiKSL6hAlgsUKVKQ997ry0cfPUvRogUBiIs7y8yZ/Zg8+T5OnTrqcIYiIiIiOcujBbIxxt8YM8AYE2uMaZ/KMT8aY1Yl2dYZY/5O3NfYGHM02f7+nnwP+cVjj7UgIiKchg2vcMd+//1bQkIa8Oefyx3MTERERPIrYzIXz/J9PPlrc2NMb6AQ0A6YaK2dkYFzngVqWWt7G2NaAw9aa7tn5r75vdX0pYiJieWllz7mrbe+d8d8fHxp2/Z17rjjRXx8fB3MTkRERCTjvLLVtLV2krV2DBCfkeONMYWA/wHDE0NlgNuNMRGJW6gxplgOpStAQIA/4eHd+OqrQZQsWRQAlyue7757hfHjWxMVtd/hDEVERESyl7fPQe4FzLXW/pP4ehZQzVrbFLgTqA58ktKJxpgexphIY0zk4cPHPZNtHnb33U2IjBxL8+bnG4hs2bKAkJD6bNo0z8HMRERERLKX1xbIKYweY609YxPnhFhrjwL9gbsTj72AtfZda22wtTa4TJlAT6Wdp1WpUpb580N48cWHMImTfU6cOMj48a35+uuXiI+PdThDERERkUvntQUy0Bv4wVq7O41jfIFoIMYzKYmfny9Dh3Zg7tzXKF++hDv+008jGDOmBUeO7HIwOxEREZFL59GH9Nw3NWYRMNlaO8MYUxr4DnjCWrs9cX9hYCtwg7V2V5Lz2gM/WmuPGWP8gCnASWttr7Tup4f0csaBA8fo2nUcP//8mztWuHAJOnX6kIYN73MwMxERETln4EA4nsJs08BACAvzfD6eduH7D8bayHTXvPCGEeTCwOVA8SSx3iQUwsmHIwsBC4wxa4AVwH7gOY9kKRcpX74Es2cPJjT0cXx9E/4pnT59jHfeuZ/PPnua2NhohzMUERGRlIrjtOJ5TVbep1/2p5E+a23LJB//A1ROtn90KudNIWHUWLyEj48Pzz9/PzfeeDWdOo1h165DACxePJEdO5bRrdtMLrvsSoezFBEREck4bxhBljygWbOrWL16LO3aNXPH9uz5nVGjGrBqVYoLjYiIiIh4JRXIkm1KlizKzJkvMH58DwIC/AE4dSqajz7qzEcfdSY6+qTDGYqIiIikTwWyZCtjDD17tmHp0pHUqlXRHV+16hOGDw9mz57fHcxOREREJH0qkCVHNGhwBRERY+jY8WZ37MCBrYwY0ZRFiybhxOopIiIi+VFgKu0gUovnNVl5n44s8+ZpWubNWVOnLqRfv3c4der8qhYNG95Px47vU6RISQczExERkfykZ0+z1lobnN5xGkGWHNep082sWjWGevWquWPr1s0iNLQBf/210rnERERERFKgAlk84sorK7FsWRi9e7dxx44e3c3o0Tfy448jcLlcDmYnIiIicp4KZPGYggULMG5cDz7//EVKlCgCgMsVzzffvMRbb93J8eMHHM5QRERExKFGIZK/tWvXjIYNr6BTpzGsWrUVgE2b5hES0oCuXadx1VW3OpyhiIhI3uGJVtN5rZ21RpDFEZdfXo4FC0J5/vkH3LHjx/fz5put+PbbV4iPj3MwOxERkbzDE62m81o7axXI4hh/fz9CQzsxZ84QypUrDoC1lh9+CCU8vCVHj+52OEMRERHJj1Qgi+NatWpIZOQ4br21vju2Y8dyQkIa8Ntv3zqYmYiIiORHKpDFK1x2WUnmzBnC0KEd8PVN+Gd5+vR/TJ7cjpkz+xEbG+NwhiIiIpJfqEAWr+Hj48OLLz7EggWhVKlSxh1fuHACYWHXceDAdgezExERkfxCBbJ4nebN67BmzVjuvruJO/bPP+sYNqwRERHTHcxMREQk9/FEq+m81s5arabFa1lrmTRpDi+88BFnz55f1aJ58y488sgEAgKKOJidiIiI5DZqNS25njGGPn3asnTpSGrWrOCOr1gxheHDr2Xv3g0OZiciIiJ5lQpk8XoNG9YgIiKcRx9t4Y7t37+ZESOasGTJZPLDb0FERETEc1QgS65QrFghPvrof7z/fl8KFw4AIDY2mk8/7cV77z3M6dPHHM5QRERE8gqPzkE2xvgDzwDDgU7W2hkpHNMY+BnYliT8hbV2jDHGAEOBh4F44FfgKWvtqbTuqznIecvmzf/QocNo/vhjlztWunQ1unWbQfXqTR3MTERE8itvbrXcs2fq+yZPvjiWlffiqfffqxekVLoaA2+/nZHcgrE20qR3H0+PIHcHLLAqjWPKAF9Za5sl2c5Vt52BNkADa+3VQCwwMkczFq9Tp04Vli8P46mn7nDHjhzZyahRNzBv3ihcLpeD2YmISH6Ul1otZ+W9eOr9pzaum9Z4b1Zy8GiBbK2dlFjsxqdxWBngdmNMROIWaowplrjvEeAda+2ZxNdvAo/mYMripQoVCmDChJ589tlAihcvDIDLFcesWQOZOLEtJ04ccjhDERERya28cQ7yLKCatbYpcCdQHfgkcd8VwI4kx+4AShljiie/iDGmhzEm0hgTefhwLvzxTTLkgQeas3r1WJo0qe2Obdz4AyEh9dm6daGDmYmIiEhu5XUFsrX2jE2cGG2tPQr0B+42xhQCDBeOPp9bHPei92GtfddaG2ytDS5TJpeuUi0ZUr16eRYuHEb//ve5Y1FR/zJu3K3Mnj2E+Pi4NM4WERERuZDXFcgp8AWigRhgD1A1yb6qwElASxjkc/7+fgwf3pnZswdTtmzCLxSstcyZM5SxY2/hv//2OJyhiIiI5BaOF8jGmNLGmOXGmFqJr9sbY0okfuxHwooXU621LmAq0M0YUyDx9L7ALKuFcCVR69aNWLNmLC1b1nXH/vxzKSEhDVi//nsHMxMRkbwsL7Vazsp78dT7N6msP5FaPKs5ONJq2hizCJhsrZ1hjKkCrATaWWsjjTFdgKcBFwkrXiwGBltrzxhjfElY5u0uEqZXbAKettamOclYy7zlP/Hx8YwY8SVvvDHzglUtbrnlf9x//0j8/AqkcbaIiIjkRRltNe1IgexpKpDzr2XLNtKpUzh79x5xx6pWbUz37jMpW7aGg5mJiIiIp2W0QHZ8ioVITrrhhiAiI8fSps35r4Xdu9cSGtqQNWsu6lMjIiIiogJZ8r7SpQP5+uuXGT26K/7+fgBER5/ggw8eZerU7pw9e9rhDEVERMSb+DmdgIgnGGPo1+8err/+ajp2HM2OHfsBWL78ff76awXdus2kUqVrHM5SRETS4s3tnD0lK62WMysrn+dLbwGdsft4ikaQJV9p3LgmERHhPPLIje7Yv/9uYsSIa1m69D3yw5x8EZHcKi+1c86qrLRazqysfJ6zswW0N/x9qkCWfCcwsDCffPIc77zTh0KFElaziI2NZvr0HnzwwaOcORPlcIYiIiLiJBXIki8ZY+jSpRUrVozm6qvP956JjJxJaGgjdu5c42B2IiIi4iQVyJKvBQVVZcWKUXTrdrs7dvjwX4wadT3z54dryoWIiEg+pAJZ8r3ChQOYNKk306YNIDCwMADx8bF8+WV/Jk26m5MnDzucoYiIiHiSCmSRRA8/fAOrV4cTHFzLHduwYQ5vvFGfbdsWO5iZiIhA3mrnnFVZabWcWVn5PGdnC2hv+PtUJz2RZM6ejeXVV6cxduy37piPjw9t2gymTZtX8PHxdTA7ERERySp10hPJogIF/Bk5sgvffPMKpUsXA8DlcvH9968xbtxtHDu2z+EMRUREJCepQBZJRZs2wURGjuOmm4LcsW3bFhESUp8//vjBwcxEREQkJ6lAFklDpUql+emnobzyyiP4+CR8uZw8eZi33mrDl18OIC7urMMZioiISHZTgSySDl9fXwYPfpSffnqdihVLuePz549h9OgbOXToLwezExERkeymh/REMuHQoSi6dRvPDz+sdccKFgykU6f3adz4IQczExHJHgMHptzqNzAQwsI8n8+l6tkz9X2TJ6cc79Ur5RbJxsDbbzt7Tlb+fjJ7Tl77N5CUHtITyQFlyxbn669fJiysC/7+fgBERx/nvfceZvr0npw9e8bhDEVELk1KhVFa8bwotbHDtMYUPXVOVv5+MnuO/g2oQBbJNB8fH/73v3tZvHg41auXd8eXLn2HESOasG/fJgezExERkUulAlkki4KDa7F6dTgPPni9O7Zv3x8MHx7M8uUfqk21iIhILqUCWeQSFC9ehOnTBzBpUi8KFiwAQGzsGaZOfZIPP+zImTP56PdRIiIieYQKZJFLZIyhW7fWLF8exlVXVXbH16z5lGHDGrNr19o0zhYRERFv49EC2Rjjb4wZYIyJNca0T+WY8saYd4wxm40xq40xS40xdRP3lTbGnDLGrEqyjfbkexBJTd261Vi5cjRdutzmjh069CejRzdjwYI3NeVCRHKFwMDMxfMiYzIX9+Q5Wfn7yew5+jfg4WXejDG9gUJAO2CitXZGCsfcCRSy1s5KfN0faG2tvd0YcyXwlrW2VWbuq2XexNNmzFhCnz5vc+LE+VUt6tW7m8cfn0LRoqUdzExERCT/8spl3qy1k6y1Y4D4NI754VxxnOhfwC/x4zJAPWPMSmPMGmPMBGNM+YuvAsaYHsaYSGNM5OHDmgcqntW+/U1ERITTqFENd2z9+tmEhjZg+/alDmYmIiIi6fHqOciJxe9Q4PXEUCRQ0Vp7HdASOAvMNebiX0ZYa9+11gZba4PLlMlHvxMQr1GzZgUWLx5Bv353u2P//beH8PCWzJ0bgsuV6s+JIiIi4iCvLZCNMaWBucBr1trFANbaGGttfOLHp4CBwFVATccSFUlDQIA/o0c/yaxZgyhVqhgA1rr47rtXefPN24mK+tfhDEVERCQ5v/QP8TxjTAXgByDMWvtpWoeSUORrDoV4tbZtm7BmTTiPPx7O8uWbAdi69RdCQurzxBNTCQpq7XCGIiLezxNtlj2ZmzffJ7O8Na+scnwEOXFliuXGmFqJry8HFgBvJC+OjTH3JBbPJE6rGAosstYe8HTeIplVpUpZfv45hJdeeohzs4JOnDjEhAl3MGvWC8THxzqcoYiId/NEm+Wsymv3ySxvzSurUhxBNsZ8mNkLWWu7ZjGHwsDlQPHE12OA8sDzxpjnE2Mx1toWgAVmGWP8ARewHuiQxfuKeJyfny+vv96Bli3r0rnzWPbv/w+AefPC2L59CU8++RllylRzNkkREZF8LrUpFqdy8qbW2pZJPv4HqJzk9YNpnDcbmJ2TuYl4ws031yMycixdu77JvHnrAPj771WEhjagU6cPaNToAYczFBERyb9SLJCttX09nYhIflOuXAm+++5Vxo79lldfnUZcXDxnzkTx7rsPctNNvXjooXD8/Qs6naaIiEi+4/gcZJH8zMfHh/7972PhwmFUq1bOHV+y5G1GjGjK/v1bHMxOREQkf0qxQDbGNDHGzEpj+8rTiYrkZU2bXsnq1eHcd9917tjevesZNqwxK1d+7GBmIiLewxNtlrMqr90ns7w1r6xKsdV0Ykvn9klCg0lYMSKOhKXVXgf8rbUuTyR5qdRqWnILay3vvvsjAwZ8SEzM+VUtmjbtxKOPTqRgwWIOZiciIpK7ZbTVdIoF8kUHGeMCClprzyZ57acCWSRn/P7733ToMJpt2/a6Y+XK1aJ798+pUqWBg5mJiIjkXhktkDM6Bzl5FZ1+VS0iWVa/fnUiIsbw+OO3uGMHD25n5MimLFz4Fhn5wVZERESyJjMP6ek7sogHFSlSkPff78eUKf+jaNGE1Szi4s4yc2ZfJk++n1OnjjqcoYiISN6UWqOQu4FZnC+KDRB1rvuXiHhOhw4tufbaWnTsOIbffvsLgN9//4bQ0F958snPqFGjucMZinivvNb+Ni/x5rbRIqmNIK8AbgNaJW43A3cm2W7JLfOPRfKC2rUrsXTpSPr0ucsdO3p0N2PG3MSPPw7H5dKXo0hK8lr727zEm9tGi6TWKOQIsNjDuYhIGgIC/Bk7tjstW9alR4+3+O+/k7hc8XzzzSC2bl1Ily5TCQws73SaIiIiuV5q6yCXyuzm6cRF8qt7723GmjVjue66q9yxzZt/JiSkPps2/exgZiIiInlDalMsDgOHMridO1ZEPKRq1bIsWBDKCy88yLlnA44fP8CECa355ptBxMfHpnMFERERSU1qBXJ14IrErQYJD+k1SBJLup07VkQ8yM/Plzfe6Mjcua9RvnwJIKHRyI8/DmfMmJYcObLL4QxFRERypxQLZGvtriTbThJWs9hlrd0F1CHhIb2kx+g7sYhDbr21PpGR47jttvru2F9/rSA0tAG//faNg5mJOC+vtb/NS7y5bbRIRjvpxQMlrbXHjTF3Ai9Ya1vmdHLZRZ30JD9wuVyMHv01Q4ZMJz7+/KoWLVs+zQMPjMLfv6CD2YmIiDgvuzvpneX8msh/AbWympiI5AwfHx8GDnyAX34ZRtWqZd3xRYveIizsOg4c2OZgdiIiIrlHhgpka20ha+2JxJdHgXJGXUNEvNJ1113F6tXh3HNPU3fsn39+IyysPhER0xzMTEREJHfITKvpc44BvkDRbM5FRLJJqVLF+OKLFxk3rjsFCiQsd37qVDRTpnTio4+eIDr6pMMZioiIeK8UG4WkxVobmzh47J/96YhIdjHG0Lv3XTRvXocOHUazffs+AFat+pi//15F9+4zqVy5fjpXEZGc0qsXpPQYkDHw9tu56z7e3AJaLa0lK1IdQTbGDE5t4/x85EwxxvgbYwYYY2KNMe1TOcYYY94wxmw1xmwyxkwzxhRJb5+IpKxBgyuIiBhDhw4t3bEDB7YyYkRTFi2aREYe1BWR7Jfal152f0l64j7e3AJaLa0lK9KaYnFzGtsSICudCLqTUFyvSuOYzkAboIG19urE+4zMwD4RSUXRooWYMuV/fPDBMxQpkrCaRVxcDDNm9OHddx/i9OljDmcoIiLiPVKdYmGtvTm7b2atnQRgjLk7jcMeAd6x1p5JfP0msAB4Op19FzDG9AB6ABc80S+Sn3XqdDNNmtTmscdGsWHDTgDWrfuKXbsi6dZtBldc0czZBEVERLxAVh7Sy2lXADuSvN4BlDLGFE9n3wWste9aa4OttcFlymgFcZFzrryyEsuXh9Gz553u2NGjuwgPv4GffgrD5XKlcbaIiEje540FsgHik7yOS/zTJ519IpJBBQsWYPz4p5gxYyDFixcGIC4unq+/foG33mrD8eMHHc5QRETEOd5YWO4BqiZ5XRU4ScLycmntE5FMuv/+5qxZM5amTa90xzZt+omQkPps2bLAwcxE8r7Uuglkd5cBT9zHm1tAq6W1ZEWGWk1n+02NWQRMttbOMMaUBr4DnrDWbjfGdAG6ALdZa88aY94CillrO6e1L637qdW0SNpiY+MYMuRTRo+e5Y4ZY7jjjkG0bfsavr6ZXhFSRETE62S01XSK3/WMMb3TOGcKEGqtfS7xWB9gkbX2pixlCoWBy4Fz84g/AWoCq40xccAmzj+El9Y+Eckif38/hg17nJYt69KlyzgOHYrCWssPP4Syfftiunb9lFKlqjidpoiIiEekOIJsjFmZyvEWaAscAupwfj7wNmutb45kmA00giyScf/+e5QuXcbxyy/r3bEiRUrx+ONTqF//HgczExERuTQZHUFOcQ6ytfa6VLbmQBQJD8ttBrYnbiKSR1SoUIo5c4bw+usd8PFJ+C/i1KmjvP32vXz++f+IjY1xOEMREZGcldWH9CxQDCgBXJZ96YiIN/D19eWllx5iwYIQKlcu7Y7/8subjBrVnIMH/3QwOxERkZyVVqvpbsaYhYnbU8n3W2tPW2uPA0dyNEMRccz111/NmjVjadu2iTu2e/evDBvWiDVrPnMwMxERkZyTYoFsjOkLvE7C6hLfAi8bYwZ4MjER8Q6lSwfy1VcvER7ejQIFEp7rjY4+wQcfPMYnnzxJTMwphzMUERHJXqmNIPcGHrPWjrXWjgMeToydY4wx7xtjpgIzcjpJEXGWMYann27L0qUjqVmzgju+YsWHDB9+LXv3bnAwOxERkeyVWoFcGVif5PV6Lp5rfArYD2gyokg+0bBhDSIiwmnf/vyqjvv3b2bEiCYsWfIOTqyrLiIikt1SK5A3Am2SvL4D2Jb4sQXirbXPWGufB14lYVULEckHihUrxMcfP8t77/WlcOEAAGJjo/n00568994jnDkT5XCGIiIilya1Ankw8K4x5gtjzAwSGnS8AmCtdVlr/c8daK2NA6rneKYi4jWMMXTufCsrV44mKOh89/dff/2C0NCG/P33agezExERuTSprYM8D7iRhDWOdwItrbXfp3YRa+2uHMlORLxanTpVWLFiFN27t3bHDh/+m1Gjrufnn8fgcrkczE5ERCRrUl3mzVr7q7V2kLX2RWttpDFGLbRE5CKFCgUwcWIvPv30eQIDCwPgcsXx1VcDmDTpbk6cOORwhiIiIpmTmUYhX6e2wxjTKBtyEZFc7MEHr2fNmrFce20td+yPP+YSEtKArVsXOZeYiIhIJqXVKOQxY8xdxpgmxphyJHsQzxhTIPHPSiSslSwi+Vz16uVZuHAYzz3Xzh2LitrHuHG3MHv2EFyueAezExERyZi0RpCnAaOA74FIElavSOpvY8yVwNXA7zmTnojkNgUK+DNixBN8992rlCkTCIC1ljlzhjJ27C38999ehzMUERFJW3pTLIKtteVIKIIBMMb8aIwpA5QnoUnIrcDynEtRRHKjO+5oTGTkOFq0uMYd2759CSEh9dmwYY6DmYmIiKQtrQLZAqWNMTcBSecY3wQUAM6S0EDkWRJaUouIXKBixVL8+OPrDB78KD4+Cf/dnDp1hIkT2/Lll/2JizvrcIYiIiIXS6tANkAzYEHiljRuAF9gJBBjrd2YYxmKSK7m6+vLK688wrx5Q6lUqbQ7Pn9+OKNGXc+hQzsczE5ERORiaRXI/YBZ1lr/pI1BOD8X2VhrNwGbjTGNcyxDEckTbrrpGtasGUubNsEVW5OdAAAgAElEQVTu2K5dkYSGNiQycqaDmYmIiFworQJ5PBBjjDlkjJmXJJ68rfQcoB0iIukoUyaQr79+mVGjuuLv7wdAdPQJ3n+/PdOm9eDs2dMOZygiIpKBh/SAu4A3k8Q6AUeSvF4INM/mvEQkjzLG8Mwz97BkyQiuuKK8O75s2XuMGNGEffs0Y0tERJyVXoG8zVq72lrrfuTcWvultTYaOLdW00bgmhTPTsYY08IY86sxZr0xJtIY0yyFY340xqxKsq0zxvyduK+xMeZosv39M/ZWRcSbNG5ck4iIcB566AZ3bN++jQwffi3Llr2PtclXlhQREfEMvzT2vQrEpLbTWls98c+jxpjx6d3IGFMCmAW0tdauNMa0BL41xlS31rp/r2qtvSPZec8C51pzlQG+stZ2T+9+IuL9ihcvwrRp/bn11vo8++x7nDlzltjYM0yb1p0tWxbQocM7FCoU6HSaIiKSz6Q6gmytDbXWJm17tTutYzNwr9bAVmvtysRzFgH/krCOcoqMMYWA/wHDE0NlgNuNMRGJW6gxplgq5/ZIHKWOPHz4eAbSExEnGGPo2rUVK1aMpk6dKu54ZOQMhg1rxK5dkQ5mJyIi+VF6Uyzczo0YX4IrgOTrOe1IjKemFzDXWvtP4utZQDVrbVPgTqA68ElKJ1pr37XWBltrg8918xIR7xUUVJWVK0fz5JOt3LFDh3YQFtacBQvGacqFiIh4TIYL5GxggPhksbjUckhh9Bhr7Rmb+F3SWnsU6A/cnXisiORyhQsH8PbbfZg2bQDFiiV8WcfHx/LFF88yadI9nDx52OEMRUQkP/BkgbwHqJosVjUxnpLewA/W2lSndpDQrCSaNOZKi0ju8/DDN7B6dTiNG9d0xzZs+J6QkAZs377EwcxERCQ/8GSB/C1QzxhTF8AY0wS4CvjFGLPcGHPuQTyMMYVJGD0elvQCxpj2iQ/7YYzxI2F0eaq11uWh9yAiHlKjRgUWLx7OM8/c444dO7aXsWNvZs6cobhcyX8hJSIikj08ViBba6OAh4APjTGrgXFAG6AwcDlQPMnhvYEfrbW7kl2mELDAGLMGWAHsB57L6dxFxBkFCvgzalRXvv76ZUqXTnge1+VyMXv2EN58sxXHju1zOEMREcmLTH548KVx45p21aoxTqchIpdgz57DdO48lqVLzzcSKVasLE888QlBQXekcaaIiEiCnj3NWmttcHrHeXKKhYhIllWuXIaffhrKK688gjEJHe9PnDjEhAl38tVXA4mLO+twhiIikleoQBaRXMPPz5fBgx9l3ryhVKhQ0h3/+edRjB59I4cP/+1gdiIikleoQBaRXKdFi7pERo7jjjsauWM7d64mNLQha9d+6WBmIiKSF6hAFpFcqWzZ4nzzzSuMGPEEfn6+AJw5E8V77z3Ep5/24uzZMw5nKCIiuZUKZBHJtXx8fHjuuXYsWjScatXKueNLlkxm5Mim/PvvZgezExGR3EoFsojkek2a1Gb16nAeeKC5O7Z37waGDw9mxYopalMtIiKZogJZRPKEEiWK8umnzzNxYi8KFiwAwNmzp/nkk65MmdKJ6OgTDmcoIiK5hQpkEbnAwYOLiYzszvLl9xEZ2Z2DBxc7nVKGGWPo3r01y5eHcdVVld3x1aunExraiN27f3UwOxERyS1UIIuI28GDi9mxYxIxMYcAS0zMIXbsmJSrimSAunWrsXLlaJ544lZ37NChPwkLu45ffhmvKRciIpImFcgi4rZ79zRcrpgLYi5XDLt3T3Moo6wrUqQg777bl48/fpaiRQsCEBd3ls8/f4bJk+/j1KmjDmcoIiLeSgWyiLjFxBzOVDw3ePTRFkREhNOw4RXu2O+/f0tISAP+/HOZg5mJiIi3UoEsIm4BAWUyFc8tatWqyJIlI3n66bbu2H///UN4eEvmzg3F5Yp3MDsREfE2KpBFxK1q1Y74+ARcEPPxCaBq1Y4OZZR9AgL8CQ/vxldfDaJkyaIAuFzxfPfdK4wf35qoqP0OZygiIt5CBbKIuJUr14IaNXoTEFAWMAQElKVGjd6UK9fC6dSyzd13NyEycizXX1/HHduyZQEhIfXZtGmeg5mJiIi3MPnhae7GjWvaVavGOJ2GiHiRuLh43nhjBiNGfHnBqhatW7/APfe8ga+vv4PZiYhITujZ06y11gand5xGkEUkX/Lz8+X11zvwww+vcdllJd3xn34ayZgxLThyZJeD2YmIiJNUIItIvnbLLfVZs2YsrVo1cMf++msloaENWLduloOZiYiIU1Qgi0i+V758CWbPHkxo6OP4+ib8t3j69DHeeecBPvvsaWJjox3OUEREPEkFsogI4OPjw/PP38/ChcO4/PKy7vjixRMZObIZ+/dvdTA7ERHxJI8WyMaYFsaYX40x640xkcaYZikc09gYc9QYsyrJ1j9xnzHGvGGM2WqM2WSMmWaMKeLJ9yAiFzt4cDGRkd1Zvvw+IiO757rW1Ek1a3YVq1ePpV278/897dnzO8OHN2bVqk8czExERDzFYwWyMaYEMAvoY62tBwwAvjXGFE52aBngK2ttsyTbuSUoOgNtgAbW2quBWGCkh96CiKTg4MHF7NgxiZiYQ4AlJuYQO3ZMytVFcsmSRZk58wXGj+9BQEDCahYxMaf46KPOfPRRZ6KjTzqcoYiI5CRPjiC3BrZaa1cCWGsXAf8CtyY7rgxwuzEmInELNcYUS9z3CPCOtfZM4us3gUdzPnURSc3u3dNwuWIuiLlcMezePc2hjLKHMYaePduwbFkYtWtXcsdXrfqE4cMb888/vzmYnYiI5CRPFshXADuSxXYkxpOaBVSz1jYF7gSqA+d+r5n8GjuAUsaY4slvZozpkTiNI/Lw4ePZkb+IpCAm5nCm4rlN/frVWbVqNJ063eyOHTiwjZEjm7Fo0UTyw1ryIiL5jScLZAPEJ4vFJc/BWnvGJn7HsdYeBfoDdxtjCqVwjbjEPy96H9bad621wdba4DJlArPpLYhIcgEBZTIVz42KFi3EBx88w4cfPkORIgUBiIuLYcaMp3nnnQc4deo/hzMUEZHs5MkCeQ9QNVmsamI8Lb5ANBCTwjWqAieBY9mUo4hkUtWqHfHxCbgg5uMTQNWqHR3KKOd07Hgzq1aNoV69au7Yb799TWhoA/76a6VziYmISLbyWKvpxGkQO4CbrbUbjDFNgJ+AmsB3wBPW2u3GmPbAj9baY8YYP2AKcNJa28sY0wXoAtxmrT1rjHkLKGat7ZzWvdVqWiRnHTy4mN27pxETc5iAgDJUrdqRcuVaOJ1WjomOPsuLL37EpElz3TEfH1/uuSeE228fiI+PVtAUySg/v1hq1NhD4cJab1wuTXy8LwcOlODgwTJYm/L/wxltNe2X7dmlwlobZYx5CPjQGGNJmB7RBigMXA6cm0dcCFhgjHEBFlgMDE7c9wkJBfVqY0wcsAl42lPvQURSVq5cizxdECdXsGABxo3rQcuW9ejRYwLHjp3C5Yrnm29eYtu2hTzxxCcEBpZ3Ok2RXKFGjT1UqVKMYsWqYYxxOh3Jpay1xMfHEhh4gKJF97BjR/JJC5njsRFkJ2kEWURyyq5dB3n88XBWrtzijgUGlqdLl2nUqXObg5mJ5A4NG26mWrWrVBxLtrDWxc6dW1m3rk6K+zM6gqzfA4qIXILLLy/H/PkhDBz4gPsb/PHjBxg//na++eZl4uPj0rmCiKg4luxiTPaUtiqQRUQukb+/HyEhnZgzZwjlyiXMFrPW8uOPwwgPb8nRo7sdzlBERDLDY3OQReRCnniwbcOGwRw/vt79OjCwHnXrDs32vDz1kJ63Pwx4220NiIwcR5cu41iw4HcAduxYTkhIAx5/fAoNGtzrcIYiIt6jceNqvPbaaO6++8EMn9OhQ1vq1w9m4MDXci4xNIIs4ghPtGdOXhwDHD++ng0bBqdyRtby8lSr6dzS0vqyy0oyZ84Qhg7tgK9vwn+xp0//x+TJ7Zg5sx+xsTHpXEFEvF27di15440XU91/4MC/XHllaX76afYl3+vIkcM891x3GjW6nKCgywgOvoKXXurLmTNn0j8Z2LDhNyZOHH3JeWTFqVOn6Nv3CapXL8bVV5cnPDwkzeOXL19EuXLeMd1GBbKIAzzRnjl5cZxePKt5earVdG5qae3j48OLLz7EggWhVKlyvmHKwoUTCAu7jgMHtjuYnUjeEhQE5cpdvAUFOZdTQEBBrrmmISVLlr7kaz355INER0ezZMlGNm7cz/z5a6lc+XJiY89m6PyNG39j0iRnCuSBA3uxefMGvv9+OW+/PZ133x3HJ5+860gumaUCWcQB3tqeOSt5eeq9eOvnLC3Nm9dhzZqx3HNPU3fsn3/WMWxYIyIivK+wF8mNDh3KXNwTSpQoyVdfzadJk+aXfK1161bTqtVdFC1a1H3tPn0GEBhYPJ0znXXo0EFmzfqUUaMmExRUjxYtbqNfv5d4991xTqeWISqQRRzgre2Zs5KXp96Lt37O0lOqVDG++OJFxo3rToECCY99xMScZMqUTnz8cRdiYk45nKGIXIqwsNe45ZaGDBkygKCgy7jzzmbExcVRrpxh+fJFAERELKd16ybUrVuRevUq0a9fF6KiMtYEuHnzloSFDWHVqmWpHvP997No3boJDRtWpXHjavTv34OoqGP07t2Jl19+hiNHDhEUdBm33toIgLi4OMaNG0bDhlW58srSNGlSk2nT3ndfb9u2zdx3383UrVuRoKDL6Nr1Qfbu/cd9r3vvbUG9epUICrqMBx9s5d6X1IYNv1KgQAEaNrzWHbvhhpvZtm0zJ0+ecMdOnz5NVNQxoqPPN4qJi4sjLi6OU6dOsX//PjZv/oO5c78hMnJVhj5n2UEFsogDPNGeOTCwXqbiWc3LU62mc3NLa2MMvXvfxdKlYdSsWdEdX7nyI4YPD2bPntSnvYiI9/vjj98oWrQY69fvZdashRftf/nlftx4461s2LCPlSu3Ub9+Y44fj8rQtSdM+Jjata/m3ntvomXL+syY8THx8fHu/d999wUzZ37E1KmzWbduN0uXbuL06dMMGtSPSZOmEhr6JqVLl2Xjxv0sWPArAKGhg5gzZxZz5qxg69YjTJs2m3HjQpk69T0Ahg9/hfLlK7B+/V5+/XUXbdrcx8GD+wHYsuUPBg0axrp1u/n99z2ULFkqxbnFJ04cp3jxkhfESpVKGNA4deqkO9a3b2dq1SrJqFGvuWMVK/pTsaI/1asXpXnzq+je/WFmzvyY06c9N6CgVSxEHHBu5YWcXJGhbt2hmV7FIit5eeK9ePI+OalhwyuIiBhD376T+fTThIcL9+/fwogRTXj44XHceONTWg9WJBeqUKESAwYMxhhDoUKFiIu7cP3zRo2asmzZL/z2WyQNGgTz5JPnmwDffvu1KY7AbtyYUJCWKVOWjz/+mm3bNvPBB2/x/PNPMX36+8yY8SNFihRhxIhXOXz4IDffXN99bnx8HGXLptzN01rLlCkTmTz5MypWrAxA7dp16NVrAO+/P55OnbrTqFFTpk9/n2XLFnLjjbfw4IMd3OcPGDCYI0cOs2TJAvbs2cWJE8c5evTiqW6XXVaJqKj/LogdPXoYPz8/ypQp54598MEX7lUsXC4Xu3cnPHxojMHf3x8fnwvHcqdNe4+CBQum+N6ykwpkEYd4oj1zeku6pSQreXmq1XReaGldrFghpkz5H7fcUo9+/d7l9OkY4uJi+PTTXmzZsoCOHd+jcOESTqcpIplQsWLlNH+4HTlyIjNnfsLzz/fEWsvTTw+kXbtHAJg3b02G7lG7dh1GjpxIhw5P0rbt9bz//nieeeYldu36i0mTpnHvvQ9n6DqHDx/i9OnTVKp0YSvmKlUuZ9euvwDo23cgtWpdxZgxQ3nhhd706PE/Hn+8B/Hx8fTt25k//viNVq3aUrny5VSrVoMtW/646D61al3F2bNniYxcRXBwMwCWLFlArVpX4evrm2JuPj4+FCxYkJiYGKKjzxAdffFKHaNGTSYgQAWyiEieY4zh8cdv5dpra9Ohw2j++GMXAL/++iW7dkXSrdsMqldvms5VRASgbNmUH8grW9bzuaTGGEP79p1p374za9dG8NhjbShQIIA2bdqleZ61FmvtBaOo9eo1onbtq92jzhUqVGLdujUZLpBLly5D4cKF2bNnF3XrNnDH//lnJ1WrVne/vuOOe7jjjnvYvn0LHTq0JSYmmuLFS7BixWJWr97hHsUdP35EigVyqVKluf/+xxg4sBcTJnzM4cMHGT9+OK+8MiLdHN9/fwKvv/58qvsHDBiidZBFRPKqOnWqsHx5GE89dYc7duTITsaMuZ5580bhcrkczE4kd9i4EQ4evHjbuNHpzBJERR1j5Mgh7oK2SpVq+PsXyNAc5OPHo2jf/k62bt3kjv3yy49s27aJNm3uA+CZZwYxZcpE5s79BpfLRXx8PIsXz2fOnK8BKFSoMGfOnObkyZPs37+PM2fO8OSTfRkzZij79u0BEh7Ke/vtMXTr1g+AkJCX2LFjG5BQgBcuXITjx6OIiYkhLi6WY8eOArBs2UImTw5PNf+wsLe55poGtG17Pb16daBXr/48/niPDH3emjdvwcGD9qKtVau7MnT+pdIIsoiIgwoVCmDChJ7cfHM9nnrqLaKiThMXF8+sWQPZuvUXOnf+mMDAculfSES8UsGChYiOPsP999/CiRPHCQgIoH37J3joofQfMC5SpCjBwdfRpcv9nDhxHGMMVapUY/Lkz2jZshUAnTp1x8/Pj7CwITz3XDf8/QtQp05dBg0KBeCmm27jyiuDqFOnDOXLV+Dbb5cwaFAoxYoF0qbNdZw5c5rAwBI888wgOnXq7r7vE0/cx5Ejh/Dz86dVq7b06fM8xhhWrlxMixZ18fPzp1mzG3niid6sWLEolfyLMH78FMaPn5Idn0qPMtZap3PIcY0b17SrVo1xOg2RC/z552QOHJgHuAAfype/nZo1e6Z5TmYfulML6Nxl584DdOw4htWrt7ljxYtXoEuXaVx11S0OZiaScxo23Ez16nWcTkMckFqr6YkTR/PDD98wffr3F53To0d7GjduluYUi7//3sy6dSn/m+rZ06y11ganl5umWIg4IKE4/pGE4hjAxYEDP/Lnn5NTPSezraPVAjr3qVatPAsXDqN///vcsaiof3nzzdv47rvBxMfHpXG2iEjesXr1cmrVKnnRtnDhTx65vwpkEQckjBxnPA6Zbx2tFtC5k7+/H8OHd2b27MGULZvQKctay9y5bzB27C38998ehzMUEckea9fuvGj0GKBPnwEpzj8+t+X0A3qgAlnEIak9fJV9D2WpBXTu1rp1I9asGUvLlnXdsT//XEpISAPWr7/4144iIpJ9VCCLOCK1L73s+5JUC+jcr2LFUvzww2u89tpj7mWeTp06wqRJd/P5588SGxuTzhVERCQrVCCLOKB8+dszFYfMt45WC+i8wdfXl0GDHmb+/DeoXLm0O/7LL+MYNep6Dh7808HsRETyJo8WyMaYFsaYX40x640xkcaYZikcU94Y844xZrMxZrUxZqkxpm7ivtLGmFPGmFVJttGefA8i2aFmzZ6UL38H578EfShf/o40V7GoW3foRcVwWqtYlCvXgho1ehMQUBYwBASUpUaN3jnSAtoT98nvbrghiDVrxnLXXde6Y7t3r2XYsEasWfOZg5mJiOQ9HlvmzRhTAtgBtLXWrjTGtARmAtWttaeTHHcnUMhaOyvxdX+gtbX2dmPMlcBb1tpWmbm3lnkTkbzCWstbb33Piy9+TGzs+VUtrr/+SR55ZDwFChR2MDuRzNMyb5Ldctsyb62BrdbalQDW2kXAv8CtSQ+y1v5wrjhO9C/nG5qUAeoZY1YaY9YYYyYYY8rnfOoiIt7BGEPfvnezdOkIatS4zB1fvvwDhg+/lr17L275KiIimePJAvkKEkaQk9qRGE9RYvE7FHg9MRQJVLTWXge0BM4Cc40xJoVzeyRO44g8fPh4NqQvIuI9GjWqSUREOI88cqM79u+/mxgx4lqWLn2P/NAESkQkp3iyQDZAfLJYXGo5GGNKA3OB16y1iwGstTHW2vjEj08BA4GrgJrJz7fWvmutDbbWBpcpE5h970JExEsEBhbmk0+e4513+lCoUAEAYmOjmT69Bx988ChnzkQ5nKGISO7kl/4h2WYPcFuyWFXgy+QHGmMqAD8AYdbaT9O4piGhwNYQsaTIUy2Qs9I2eu3avkRH/+N+XbBgFRo3npDmOcuXP8CFP2f6cv31X6V6/IoVHUj4WTKBMUVo3nx6mveIiOhKXNxR92s/v1I0bfphmueopbVzjDF06dKKpk2vokOHUWzcuBuAyMiZ7Ny5hm7dZlCt2rXpXEVEMqtdu5Y0btyMV18dcdG+Awf+5aabrmH8+I9o3fruS7rPkSOHCQ19iUWL5hETE0OhQoVp1eouBg8Oo1ChQumev2HDbyxZMp8+fQZcUh75jSdHkL8lYf7wuRUpmpAw+vuLMWa5MaZWYvxyYAHwRvLi2BhzT2LxTOK0iqHAImvtAQ++D8klPNUCOStto5MXxwDR0f+wdm3fVM+5uDgGiE+MXyx5cQxg7SlWrOiQ6j2SF8cAcXFHiYjomuo5amntHa6+ugorVoyiW7fzSwUePvwXYWHN+fnnMbhc2deERsTbHDs2nW3bqrFxow/btlXj2LG0BwJyWkBAQa65piElS5ZO/+B0PPnkg0RHR7NkyUY2btzP/PlrqVz5cmJjz2bo/I0bf2PSJC34lVkeK5CttVHAQ8CHxpjVwDigDVAYuBwonnjoGKA88HySpdzOfQe0wCxjTCQQAZQDUv9uL/map1ogZ6VtdPLiOL14guTFcdrx5MVxenHgouI4vTiopbU3KVQogEmTejN9+gACAxNWs3C54vjqqwFMmnQ3J0+qu6HkPceOTWffvh7Exu4CLLGxu9i3r4ejRXKJEiX56qv5NGnS/JKvtW7dalq1uouiRYu6r92nzwACA4unc6ZcCo+ug2ytXWitvdZa28Ra29xau9Ja+4+1trK1NjLxmAettaWttc2SbC0S98221l6XOLe4ibW2m7U29e/ckq95rgVyzreN9mZqae19HnroBlavDic4uJY79scfc3njjfps26YRd8lbDh58mSSrxQJg7WkOHnzZI/cPC3uNW25pyJAhAwgKuow772xGXFwc5coZli9fBEBExHJat25C3boVqVevEv36dSEq6liGrt+8eUvCwoawatWyVI/5/vtZtG7dhIYNq9K4cTX69+9BVNQxevfuxMsvP8ORI4cICrqMW29tBEBcXBzjxg2jYcOqXHllaZo0qcm0ae+7r7dt22buu+9m6tatSFDQZXTt+iB79/7jvte997agXr1KBAVdxoMPtnLvy0vUSU/yLM+1QM75ttHeTC2tvdMVV1zGokXDePbZe92xqKh9jB17C99//zouV2q/kRDJXWJjd2cqnhP++OM3ihYtxvr1e5k1a+FF+19+uR833ngrGzbsY+XKbdSv35jjxzP2EO2ECR9Tu/bV3HvvTbRsWZ8ZMz4mPv781+93333BzJkfMXXqbNat283SpZs4ffo0gwb1Y9KkqYSGvknp0mXZuHE/Cxb8CkBo6CDmzJnFnDkr2Lr1CNOmzWbcuFCmTn0PgOHDX6F8+QqsX7+XX3/dRZs293Hw4H4Atmz5g0GDhrFu3W5+/30PJUuWIjw85FI/hV4nf3wHl3zJUy2Qs9I2umDBKpmKJ/DNVNyYIpmKQ8IDeZmJg1pae7MCBfwZObIL33zzCqVLFwPAWhfff/8a48bdxrFj+xzOUOTS+ftXzVQ8J1SoUIkBAwbj6+ub4oNzjRo1ZdmyX/jtt0iKFCnCk08+TZUqlwNw++3XEhR02UXbOWXKlOXjj79m6dKNNG16A88//xTt2rXk1KmE6XIjRrxKRMQybr65PkFBl3HttVewcOGPrF+/NsVcrbVMmTKR/v0HU7FiZQBq165Dr14DeP/98e58f/stkmXLFhIQEMCDD3agYcOEh30HDBhMzZpXsmTJAj77bAonThxn58681/JeBbLkWZ5qgZyVttGNG0+4qBhObxWLhNUqkhfDqa9i0bz59IuK4fRWsWja9MOLiuH0VrFQS2vv16ZNMJGR47jppiB3bNu2RYSE1GfDhrkOZiZy6cqVC8WYCztIGlOYcuVCPZZDxYqVSaElg9vIkRN54onePP98T267rTHffDPTvW/evDVs3Lj/oi252rXrMHLkRObMWcHvv0e6i9ldu/5i1KjJF5y7Zcthli7dmGIuhw8f4vTp01SqdOEPEFWqXM6uXX8B0LfvQF57bTRjxgylefOr+OijybhcLmJjY+nZ8zHuvfcmliyZz9mzZ6lWrcYFI9p5hSeXeRPxuHLlWnikgKpZs2e6y7oll96SbilJa0m3lKS3pFtK0lvSLSWe+jx76j55UaVKpfnpp6EMG/YFoaGf43K5OHnyMBMn3sVtt/WnXbth+PkVcDpNkUwrUSLhWf2DB18mNnY3/v5VKVcu1B33BsYY2rfvTPv2nVm7NoLHHmtDgQIBtGnTLs3zrLVYa/HxOT+eWa9eI2rXvto977dChUqsW7eGe+99OEO5lC5dhsKFC7Nnzy7q1m3gjv/zz06qVq3ufn3HHfdwxx33sH37Fjp0aEtMTDTFi5dgxYrFrF69g4IFCwIwfvwItmzJex08NYIsIpJP+Pr68uqr7Zk3bygVK57/TcH8+WMYNeoGDh36y8HsRLKuRIkO1K69k6AgF7Vr7/Sq4jgq6hgjRw5xF7RVqlTD379AhuYgHz8eRfv2d7J16yZ37JdffmTbtk20aXMfAM88M4gpUyYyd+43uFwu4uPjWbx4PnPmfA3w//buPbjK+s7j+PsTAkGQy7aIOmLkOjRFrULWYjugtkpb1o6rNS22qKhoFeu17nGh8xQAAAzcSURBVLhL7S5W8MrFaitIHacLaG/WKepUV2sVtVwEdcpaLUszlAwgclMgNYQT8t0/ziEejyE5ET1PcvJ5zZyB5/v8nuf5JvNM8uXH78Ihh/Sgru49amtr2bx5E3V1dVxyyVXMmvUjNm3aAKQn5c2dO4vJk68GYPr0/6C6+v+AdAHeo0dPdu3aSX19PQ0NKd59N70+wksvPce8ebM/pu9U++IeZDOzTmbs2GNZuXIOkyffw5NPpscprl+/khkzTmTixJ9RWZlfT5SZta5790PYs6eOc875Ert376KsrIwJEyZRVdX6/ImePQ+lsvJkLrroHHbv3oUkjj56IPPm/YJTTz0DgPPPv5TS0lLuvPO/uP76yXTt2o2KiuOYOjU9xGTs2NMZPnwEFRX9OPzwI1m8+AWmTp1Br169GT/+ZOrq3qN3775cc81Uzj//0qbnTpp0Ntu3b6W0tCtnnHEmV175b0hi2bIlnHLKcZSWdmX06DFMmjSFpUuf/4S+e8lRRCSdwydu1KihsXz5rKTTMDNrVxobG7nnnsf5wQ8Wkko1NMXHjLmMqqq76dat9V26zA7WiSe+yaBBFUmnYUVk3bo3ee215t+pyy/XKxFR2do9PMTCzKyTKikp4dprz2LJktsYNOjwpviLL87n9ttPYtOmN1q42syseHmIhVmOLVuWUFOziPr6bZSV9aO8fGK7mRjW1txyt7RubaUM65wqK4fx8suzueKK+3jkkT8BsGnT69x114mce+5cvvCFi1qcoW9mVmzcg2yWZcuWJVRX30d9/VYgqK/fSnX1fWzZkvzuY23NLbc4hvRW1q+8clUBsrWOpk+fnjz00A3MnTuF7t3Tq1nU1e1l4cJLePDBidTV7Uo4QzOzwnGBbJalpmYRjY31H4g1NtZTU7MooYze19bccovj1uJmkrjkknEsXXoXFRXvr9O9cuXD3HrrSNavb37jATOzYuMC2SxLff22NsULqT3nZsXl2GOPYdmymVx00elNsa1bq7nzzpN59tm76QyTu82sc3OBbJalrKxfm+KF1J5zs+LTo0cZ99//PRYsuJ5evdKrWezbl+I3v7mOuXPPorZ2e8IZWjHxP7rs4/JxvUsukM2ylJdPpKSk7AOxkpIyystbX6/yk9bW3HK3sm4tbtacCRPGsmLFbEaOHNIUW736cWbMOIG1a19MMDMrFvv2dWHfvlTSaViRSKXqSKW6HvR9XCCbZenf/xSGDJlCWdlhgCgrO4whQ6a0i1Us2prbqFH3fqgY9ioW9lEMHXokS5bcztVXf70p9s47G5gz51R+//vpNDbuSzA76+jefrsvO3a8TURj0qlYBxYR7N37Hm+9tZGamv4HfT9vFGJmZnl74omXmTz5Xnbs2N0UGz78S1x88SL69Dkywcyso5IaGTx4A717/yPpVKyDS6W6UlPTn127eh+wTb4bhbhANjOzNtmwYRsXXDCbl156fyORXr0OY9KkBYwY8dUEMzMza5l30jMzs0/EgAH9ePrpW5g69ZtNG4js3r2Ve+/9Go8+eqPHk5pZh+cC2czM2qy0tAvTpn2bp566mSOO+Kem+NNP38nMmWPYtu3vySVnZnaQClogSzpF0quSVktaJWl0M20k6RZJayS9IWmRpJ6tnTMzs8I77bTjWbVqDuPGndgUW7duBTNmnMCrr/42wczMzD66ghXIkvoCjwJXRsTxwA3AYkk9cppeCIwHToiIzwIp4I48zpmZWQL69+/LY4/9kNtuu5DS0i4A1NXtZP78c3n44SmkUnsSztDMrG0K2YP8FWBNRCwDiIjngbeAL+e0+xZwf0TUZY5/DJyXxzkzM0tISUkJ3//+2Tz33K0MHPj+EksvvDCX22//PJs3/zXB7MzM2qa0gM8aDFTnxKoz8ZbaVQOfktSnpXMRsTP7JpIuAy7LHNZ36/avrx9k/tax9QO8J3Pn5ncgIRs3rmbatIqk0wC/A+Z3wGB4Po0KWSALyF1NvoEP92LntmvI/FnSyrkPiIj5wHwASavyWdLDipffAfM7YH4HzO+ASVqVT7tCDrHYAJTnxMoz8ZbalQO1wLutnDMzMzMzO2iFLJAXA8dLOg5A0knAZ4A/SvqTpGGZdguByZK6ZY6vAh6N9I4mLZ0zMzMzMztoBRtiERE7JVUBD0oK0sMjxgM9gGOAPpmmC4ChwMuSGoA3gO/lca4l8z+2L8Q6Kr8D5nfA/A6Y3wHL6x3oFFtNm5mZmZnlyzvpmZmZmZllcYFsZmZmZpbFBbKZmZmZWZaiLpAlnSLpVUmrJa2SNDrpnKywJHWVdIOklKQJSedjhSfpu5L+nPkZsFrSlKRzssKSdEvmHXg58zvhiqRzsmRIGiFph6RpSedihSXpfyWtlLQ88/ljS+0LuVFIQUnqCzwKnBkRyySdCiyWNCgi3ks2OyugS4EAliediBWepC7AMOCLEVEr6Sjgb5IWR8TGhNOzwnkHqIyIlKTDgHWSnoqIdUknZoWTqQt+Cvwi6VwsEb2Az0VEYz6Ni7kH+SvAmohYBhARzwNvAV9OMikrrIi4LyJm8eFdHK0TiIh9EXFDRNRmQtuBvUCXBNOyAouI2RGRyhwOJL3B1I7kMrJCk1QC/DcwFdiacDqWjE8BSyS9JunXkk5oqXExF8iDgeqcWHUmbmad093AryKiJulErLAkDZO0FvgfYGJE7Ew6Jyuo6cAzEbE06UQsMYdHxBhgFOnN6/4gKXeH5ybFXCCLD/caNlDcX7OZHYCk6cBR5Le5kBWZiFgbEcNI/y/iAkkjks7JCkPSN4DyiPhJ0rlYciKiLvNnY0Q8BLxCerRBs4p2DDKwATg9J1YOPJJALmaWIEkzgSHANyJib9L5WHIi4jVJy4DTgL8knY8VxNeACkn756IMgPSEvYioSi4tS1gXYNeBThZzb+pi4HhJxwFIOgn4DPBMolmZWcFIKpE0DzgaqHJx3PlIOk7SNyUpc3wU8HlgVbKZWaFExOSIGBURoyNiNPAA8ICL485D0j9LGpV1PB6oID3kqllF24McETslVQEPSgrSwyvGR8S7CadmZoUzHvgu6WLopUyNBHBTRPwhsayskNYDlwM3SkoB3YAfRoRXtjHrPGqB2ZKOAOpJT9Id11JNqIgoVHJmZmZmZu1eMQ+xMDMzMzNrMxfIZmZmZmZZXCCbmZmZmWVxgWxmZmZmlsUFspmZmZlZFhfIZmYdkKRxklZIqpW0RtLVOeeXS2rTroFZawVXZpbHbMu1P5fkncrMrCi4QDYzS5ikmyRF1ueqnOOQNDGr/Ujgd8BPgWNIr/X875Iuy/N5D0lqyPqkJDUCV7RwzWBJz0jaI6la0oSD+6rNzNovF8hmZsm7A+gFjAQCmAd8CxiYif8V2JnVvgp4MiIWRMT2iHgemAl8J8/nnQ90z3zKgBGkF89f3FxjSWXAk8Am4FjgR8DPJY3J+ys0M+tAinYnPTOzjiIiUkBKUjfgnYhISXoYOD0i1kvqB2zNuqQE2Jd7mzY8rxFoBMjsLPVrYFpEbDzAJWcB/YFLM9t1/03SWOA64MV8n2tm1lG4B9nMrP3oB7yWHZB0KLAHWJcVfgQ4U9K5knpKGg1cD/wq3wdJKpH0HWAF8FhE3NFC85OApZnieL9nM3Ezs6LjHmQzswRJ+jvpccTZsf29wc9lhTdn5tCdHRG/y4wBvgVYBGwGfgLMzeN5PYH/JD1M423gwswQjZb0BrbnxLaRHv5hZlZ0XCCbmSUoIgZ+xOseAx77CNf9Q9Ja4OvAGgBJub8L1gPnZR1v5MO9xf0ycTOzouMC2cysnZBUBSzMCb8QEeMO0L4LoAPc7l+AuuZORMQDmes3H+DaUuDTwC8zx68D10nqHhF7MrEzgL8c4Hozsw7NBbKZWfvxBDA0J1YvSRHR3CS8eqBLC/f7MXDtgU5GxBHNxSWdwAfHQj9OejjGzyTdDHwR+DZwegvPNjPrsFwgm5m1A5mxyAOyQgE0AN2ALZIGZfXephtElEq6Bzg0Ii7O3KcnUAt8NiLebOWZea18ERF7JX0VmE+6N3kjcEFEeAULMytKLpDNzNqB3LHIkkYANwKfA87LLY6z7CW9lvF++3+uN+T56Ergz3nkt470sAozs6LnAtnMrB2Q9GnSG3acRnr8cAXpjTzmAAMk7QI2Zg+1yEyuawC6Z020218sh6QuEZG7XnKuLhzgd4GkxsyayWZmnYrXQTYzS5ikm4A3gZtJD6m4kvTSaiNJ76B3Den1ikdlXTMQSJHuZT4n8/cU6bHCAGuBJXk8fgXpyXzNfZqdHGhmVuzU/LwPMzMzM7POyT3IZmZmZmZZXCCbmZmZmWVxgWxmZmZmlsUFspmZmZlZFhfIZmZmZmZZXCCbmZmZmWVxgWxmZmZmluX/AUXePQk1nC/sAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Iris-Setosa 아님\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
    "plt.xlabel(\"꽃잎 길이\", fontsize=14)\n",
    "plt.ylabel(\"꽃잎 너비\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 활성화 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAEYCAYAAADMNRC5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8VFXawPHfmfSEkABpFOlVQBCQrkSqiIKKWBYLNuwd39VFd10su64FRUXBuhTBRUVRQREkiALSkSZVEEJIhfQ6c94/7kyYJJNkkkxLeL587mdm7px778lNmDvPPc85R2mtEUIIIYQQQghnmLxdASGEEEIIIUT9IQGEEEIIIYQQwmkSQAghhBBCCCGcJgGEEEIIIYQQwmkSQAghhBBCCCGcJgGEaFCUUrOVUkftlr/ZvfeNUurZGu6vRtsopRKUUvdU8f5tSqkzdssmu/filVI51ew/VimllVKDnK2TEEII91NK/aPc5/tndu9NUUrtrmb7AdbP9+bur60QdSMBhGgQlFLDlFJHgcvLvTXVGki0qmb7OUqpBU4e65D1Q962VBowlKe1/khrHWm39Hd2W6trAAuwr4bbCSGEqCHrTSFdxTLUVlZr/c9yn+/X1vBw1wDpwCmX/hBCuIG/tysghCtordcCbZVScUB/IBzIAH7WWmcDKKWq2kUcsNtazh9oZF1f+n9EKXU+cD7wd8oG38VKqWuB1ZXtXCnVGvitiuM/C+yoqoJKqQuAF4E04F2l1K1a68KqthFCCFFn/wBeq+S9PGsQ8U0V299e3QGUUiOBh4Fs4BWl1BNaa0uNayqEh0gAIRoMpdQY4GtgP8ZdnNZApFJqgNb6oLVYsFIqEsjVWhdbtwsH4oFG1uDhMut+bDZaH2OAHlVUYX1lb2it/7TWxR8YCbTHCAR+1FqnWesRb32MNDbRmdbXJuAe4GXgTevjMmCLUmo68LWWGSGFEMJdirTWVaWX/ozx+R4CjAZaAknAD7btlFJTAJP1891sd2MrEPgb8CTwGLACWA70V0o9bb05JoTPkRQm0ZC0A7YBF2mt4zFaIiwYrQs2fwVOAxPt1s0CEoB8YKbW+huttdJaK+BbWyGtdQJGC0AocCtwPzAJSNFaP6u1PllV5ZRSTYGdwN3WfQwF9lnvPNmEWeuXbN3mQeBP4AngZq31k1rrdGAYMA/4EDiulBrtxPkRQgjhBkqpDsBB4GqMz/dxwGGllP1Np24Yn++brdu8AJzEuB6N0VrP1lr/AfTFCEq+VUodVEr18txPIoRzpAVCNCRzML54z1ZKXQ28g/HB/Iv1zhDAP7XWzwIopUKBmcDFwGCMAGKlUupr4BGt9WEHx/gbMAYYqrVOVEr1AX5USv2htV5hLeOvlAoGirXWZrttrwUsWuurbSusnaYfA1ZZV+VqrRvZbbMZeARYprUusq3UWpcALyul3sRo0fjJ+dMkhBDCxe4Atmutp9hWKKU+Ae6zLgB7tNb2AcU667LSPl1Ja50HPKWUehG4lKrTX4XwCmmBEA2CUmohsAt4DuMDNxK4AXgP2I7R8lDeMiAK6K+1TrE2KQ/FuPMTX8mhzgfWaK0TAbTW2zD6TthfFN7ECEYeLLftfqC1tcN3kFKqvbWuv1f2c2mtN2qtP7MPHsq9X2BtMSmobB9CCCHq5F9VdKK+yVrmd6CXUqq3UirQ2vJwEVV/vn9nXRz2ddBaZ2utl0mKqvBF0gIhGor7gACMlKUSIM96l76UUmoIRvOxzdW2PFQba4vBS3arfgKO2L1eDHxkHfHpEEbA0YuyneTu1Vq/W76CWuu1Sqn7rPtvY63LMuCf1iJnMO5G2er7MUaqlLPGaq2/q0F5IYQQVRsL+FmfXwW8CnSwe78AQGs9TynVBCOttAVGa/gHwFvWcqewpi6BMboTRiqqs7pprSsNRoTwNCWBrWhIlFIvUfWX7le01q+U28YEPAD8BePC0Azjy/wmjD4R35crPxi4E7gNo0/Eh7Z0J+tFYbGjAMJu+8aAAmz/+bT9c611rl29yrcSdscYrSkKyCz3nlnuVAkhhHtYR9t7S2sdV0WZCMp+tts/mrXW+dZyfhjXAXuXY/Rtiyq/3/I3xITwNmmBEA3NDIxRihyZx9nhWe29gtEZ+mGMFocsoDlwPbBMKXW11nq5rbDWer1SKg8jgHgDCLL2hXDWASDa+lyVe8y11dHarF2maVspZetTYZYLihBCuEclX/BN1vcqfHfSWpdY+76d4ezntiq3jz1Y013L9Y+zHdNi21dd6y+Eu0kAIRqaV4HJGMO4OrLSwbrLgde11l/YrTsG/EcpNQKjCXu5Umon0MX6vu3/zp8Y800kA/bbV2eC1rrMuOHWYVyrGktcCCGEZ6ym8hSj4vIrlFLdgKPWl7201rvLvT8FmObC+gnhVRJAiIZoodba6dmhMcbdflApdQhjLoccjDkfrsG4gLxtLTcUIxdWY9xhynfQz2KEk8cMs44Hbs9R64gQQggPsw4FXiPWFgiAcAef76F1rpQQPkQCCNEQ3WE3MkZ5m7TWw8utm4bR4vAUxgRvTTGaobcC19jSl8p3uK6jxZWsz3XhMYQQQnheZZOK7vFoLYRwI+lELYQQQgghhHCazAMhhBDCpZRSAUqpaUqpYqXUDZWUiVVKzVFK7VNKbVJKrVNK9fR0XYUQQtScBBBCCCFc7S6MvkIbqyjTB/hea91Na90f+BJjEAQhhBA+zqf7QERFRem2bdt6uxrk5uYSFhbm7Wr4FDknFck5qUjOSUW+ck62bt2aprWOrr5kzWmtZwMopa6sosyKcquSqOSapJSaCkwFCAkJ6Xveeee5qKZ1Y7FYMJnkPpw9OScVyTmpSM5JRb5yTg4cOODUtcGnA4i2bduyZcsWb1eDhIQE4uPjvV0NnyLnpCJfPifmfDO7J+wm6qooWtzTAmUqP7y5e/jyOfEWXzknSqlj3q6DjVIqFmMOlzscva+1ngvMBejXr5/2hesC+M7v0pfIOalIzklFck4q8pVz4uy1wfuhjhDC7dK/Sef0D6dJ+jDJY8GDEM5QSjUDlgPPaq3Xers+QgghqufTLRBCCNdoOrYp3RZ0w6+Rn7erIkQppVRzjHlY/qO1/sTb9RFCCOEcCSCEOAf4N/IndnKst6shzmHWloZlwBSt9UGlVBuM4OEZrfXn3q2dEEKImpAUJiGEEJ4QCrQBIqyvXwVigSeUUhuti6QwCSFEPSAtEEI0cLvG7yIwNpC2z7UlKC7I29UR5xCtdbzd8+NAK7vX13qjTkIIIepOWiCEaMDyj+aT/nU6yQuT8QuT/g9CCCGEqDsJIIRowFI+SQEg6qoo/MOlwVEIIYQQdScBhBANlNaa5AXJANKBWgghhBAuIwGEEA1Uzo4c8vblERAVQJPRTbxdHSGEEEI0EBJACNFAJS80Wh+ir4/GFCD/1YUQQgjhGvKtQogGSJt1af+H2JskfUkIIYQQriMBhBAN0JmEMxQlFRHcIZjGAxp7uzpCCCGEaEAkgBCiAbLvPK2U8nJthBBCCNGQSAAhRANjzjeT+nkqIKMvCSGEEML1JIAQooFJ/zodc7aZ8IvCCe0c6u3qCCGEEKKBqXMAoZQKUEpNU0oVK6VuqKSMUko9p5Tar5Taq5RaoJQKq+uxhRAV2UZfktYHIYQQQriDK6amvQvQwMYqytwKXA701lrnK6U+Al4CHnDB8YXwCcGnTsHSpd6uBm0v9iPEEkh0+BZYutmrdYnavRtOn/ZqHZyhNVgsUGIxYbYoSsx2i8VU5rnZokrLa4znWoNFW59b15W+Lrfu2OGj/PTzT2jAYlFoQGvH29jXz9FzqLxche2QvjBCCCFco84BhNZ6NoBS6soqil0PzNFa51tfvwGsRgII0VBoTZ/77vOJL8vh1oVvvFwRoIcL9pFPMGlEkUkEWTSudMmhEQUEV7rkE1L6vAT/ckuAC2rqrEs8eCwhhBDC9VzRAuGM9sBhu9eHgaZKqQitdaZ9QaXUVGAqQGxsLAkJCR6qYuVycnJ8oh6+RM5JWcpsZpg1eEgdOtTLtfEd5pIS/Pwdf8zkmENILIzlREEsxwvjOFkYQ2pxU9KKI0krblL6mGv2XD+OAFWMvzLjp8z4Wxfbcz+MR5OyYEKjlEZxdkFpLMqCRWm0MhOoFYEaFJDrX0J2QDEWZcGMBYtJYzGZsSgNaHrmhBr7UZotEVnk+ZeAdZ9naVrnB9MpLwQFZAQUsy0iu/S9UtZthmREEGb2QynNzvAcTgUVld3ffreeSiGEEA2YpwIIBZjtXpdYHyv0wdBazwXmAvTr10/Hx8e7vXLVSUhIwBfq4UvknJRTVGQ8+vsTvW6dV6qQ/0c+v9/yO83vbE7crXFeqUN5P/6YQNu28ezbR5nlwAFIT3duHwEBEB0NkZHQuHHlS1gYhIRAcLCx2D+3X4KCIDAQ/P3PLn5+YDIBBAABFJQUkJqbQUpuCql5qcZjbiqRwZHc0ecOAPKK8+g3tx9nCs5wpuAM+SX5Zeo976p53NzrZgBm/TqLh7972PHPZwpg3TNFpa8veOcCdqXsIsAUQLB/cJnl1l63Mv2S6QDsTd3Lk6teqVAm0C+QQL9AHh7wMLGNjH4wKw+v5FDGCQL9AgkwBRDgF8DkC5z/PQohhBD2PBVAnABa271uDeQAZzx0fCHcy2IxHv38vFaFlEUpZP6cSdB5QV4LIFJSYNMm+PVXY1m/fii5uY7LBgVB69bQpo2xtG4NcXEQE2Ms0dHGY+PG4KqpLMwWM8m5yRzKSuRE1onS5eGBD9MivAUA93xzD3O2znG4/YVxF5YGEMH+wexP349FG797f5M/TYKbEBkcSWRwJOFB4aXbDWg5gL9f8ncaBTYi8WgifXr0oVFgIxoFNiIsIAytdel8Hb/e+SuBfoH4mar+Wzo/+nyW3bjMqZ97dIfRjO4wusy6yUx2alshhBCiPLcEEEqpZsAyYIrW+iAwH7hTKbVYa10EPAh8oXX5Ln9C1FO2AMLkvZGRWz7YkqBWQYR28VzKT1YWrFkDP/xgLAcOlC/hT1wcdO8O3bqdXbp2NYIFd8xxl1WYxeGMw5iUiV5xvQDYn7afMQvGcCLrBGZtrrDNmI5jSgOIxkGN8Tf5ExMWQ3RoNDFhMaXPOzbtWLqNSZnYfe9uIoIjiAyOJMQ/pNJJ+wa0GsCAVgMASChOIL5XfKX1DwkIqe2PLoQQQniEu1ogQoE2QIT19TygI7BJKVUC7EU6UIuGxGz9UurFAMI/3J+4W9zf8pCWBl99BUuWwOrVUFJy9r2wMLjoIujfHwYMALN5PZMmDXZbXX7+82c2HN/AntQ97E/fz6GMQ6TlpQFwRecr+PrGrwGICo3iWOYxAKJDo2nVuFWZpU1Em9J9Pj/8eV4a+ZJTM3h3i+7mhp9KCCGE8G0uCyC01vF2z48Drexem4Hp1kWIhsfLLRD2KTDuUFwM334Lc+fCypVl46UhQ2DUKBg92gge7PtMJyQUOd5hDeQU5bA9aTtbk7ayM3knLwx/obS14N0t77Jw18Iy5YP9g2nfpH2ZoKBpSFMOPHCA8yLOI9g/uMrjBfoF1rnOQgghREPmqT4QQjRsXuwDoc2azb02EzEkgo6vdcQvzHV1OHUK3noLPvjAeA5GgDBmDFx7LUyYYPRVcKUzBWf4cPuHbE3ayrakbexP24+2G2XouvOvKw0gruh8BU1DmnJ+9Pl0i+pGx6YdaR7eHJMqG8gppejUrJNrKyqEEEKcoySAEMIVvJjCdHrNafL25GEpsGAKdc3xjx6Fl182AofCQmNd165w111wyy0QFeWSw5CSm8Ivf/5CTlFO6YhFAI+vfLz0eYApgB4xPejbvC8XNr+QHjFnZ5e4occN3NDjBtdURgghhBBOkQBCCFfwYgpT8oJkAGInx9Y5jSk5GZ55Bj788GxMdPXV8MgjcPHFde/0nJKbwqojq1h1ZBU///kzBzMOAtAyvCU3XXATSikigyN5auhTtI1sS9/mfekR04Mg/6C6HVgIIYQQLiMBhBCu4KUUJnOembQvjE7DsZNja72fwkJ44w14/nnIzjZ+jJtvhr/+1RhByRVmb57N/cvvL7MuNCCUQa0GMbT1UEosJQT4GTNCvzjiRdccVAghhBAuJwGEEK7gpRSm9K/TMWebCb8onNDOtRu+de1auOMOOGydK/6KK+CVV6BLl9rVKTknmWX7l7HswDI6WDoQTzwAvWJ7EewfzCVtLmFU+1EMazOM3nG9S4MGIYQQQtQPEkAI4QpeSmFKXmhNX7qp5q0P+fkwfTq8/jpoDeefDzNnGqMp1dThjMMs/X0pX/7+JeuPry/t9Nw3sm9pmYGtBpLxfxkyz8E5QCkVADwM/Au4WWu92EEZBcwArgPMwDbgbq11JVMPCiGE8BUSQAjhCl5IYSpKKyJjRQb4Qcz1MTXadts2mDwZfv/dqPL06fD00xBQi8aA+7+9n9lbZpe+DvQLZFT7UVzV9SoiUiNK1/uZ/AgxSfBwjrgL0MDGKsrcClwO9NZa5yulPgJeQuYIEg2A2Qxff20MSHHkSEvi4oyBKI69eIyS0yXVbm9jCjXR7p/tSl/btm/9VGsCmhof2KfmnSJ3V83ibkfbx94cS6MLGgFwevVpMr7LqNE+HW0fOTySZmObAZC3P4+k95OMwsfh8LeHq92no+1DOofQ4i5jJD5zrpmjzx6tUT0dbV/ZeXaWS35PDs6JN35PzpIAQghX8EILROqSVHSJpullTQmMdX7ugoUL4c47oaDAuKDNm2fM3+CMjPwMluxZwoBWA+gd1xuA/i37s2DXAsZ1GsfVXa/mso6XER4UDkBCQkJNfyzRAGitZwMopa6sotj1wBytdb719RvAahwEEEqpqcBUgNjYWJ/5u8rJyfGZuvgKOSewfn0zZs/uQGKiLa20E2+9pbnkklSe2fsHfqk12FljOHbpsbOvZwHJcLzvcbDNG/o+sK5mdXS0/fFGx8H2XXQR8EEN9+lg++Mnj4PtvtEm4BW78hyvfp+Otu8HBzodMNZllt2nUxxtX8l5dpqLfk/lz4lXfk9OkgBCCFfwQh8I2+hLMZOda30wm+HJJ43+DWAEEbNmQUg1HxoWbWHNH2t4f/v7LN23lEJzIff2u5fZ44xWh+t7XM/1Pa6vdoI2IcppD9jfbjsMNFVKRWitM+0Laq3nAnMB+vXrp+Pj4z1WyaokJCTgK3XxFef6OfnwQ6M1V2to3x5u755OxzUH+DEvlrlr29MxpoSH/lZCZKRz+zMFm2gVXzovLyf/eZKSrBJajG2Bf4TxFS71sVTyr8yvbBcOOdo+akJUaV+6zOBMMrtkVrWLChxt33hAYyIvMX7Y/Db5pCojejpy+AjtO7Svdp+Otg9uG0xMvHHdM+ebSfxPYo3q6Wj7ys6zs1zxe3J0Trzxe3KWBBBCuIKHWyDy/8gna30WplATUVdVPylDdjZMmgTff29MBDdrFtxzT9XDsiZmJfLxjo/5YPsH/HHmDwAUilHtRzG83fDSchI4iFpSGH0fbGxXa+9M5y5EHc2fbwxIAfDss0ZqaNqiYn7/upC7ri7g1yPwwc4WrP4EtmyBZjXLGAGgxd0tKqyLvqpus3k62j5iYAQRAyMclHaOo+1D2oXQ+onWABxJOELr+NY12qf99jZ+IX4V1tVEZds7Os81UZvfU3XnxFO/J2dJACGEK3i4D0TKJykARF0VhX+jqv8bnzkDY8fCxo3GrNGffQaXXFL9MV5Y9wLvbHkHgNYRrbm99+3cduFttI6o/Ye1EHZOAPZ/TK2BHOCMd6ojRO0dPAj33ms8f+UVeNw6F2azcc1gLnQf2Y41kTBqFGzdakzK+fnndZ9bRwhvkQBCCFfwYAqT1vrs5HHVjL6UlmaMqrR9O7RpA6tXQ4cOFcsVlBSwaNci4hrFMbbTWAAe7P8gqXmp3HnhnYxsPxI/k2fnuBANi1KqGbAMmKK1PgjMB+5USi3WWhcBDwJfaK21N+spRE0VF8Nf/gK5uXDDDfDYY2ffC2gaAJ2Mu+chGDdwevWCpUvh/feNQEKI+kgCCCFcwcMpTF3/25XUz1JpMrJJpWWSkmDkSNi7Fzp1glWroHW5xoPT+ad5e/PbzPp1Fql5qfRr0Y/LOl6GUopu0d1YMmmJm38ScQ4JBdoAtvbyeUBHYJNSqgTYi4zAJOqhd94xUpJatzaeV9Wq0LatUWbyZJg2Da6+GqKqz0IVwudIACGEK3gwhUkpReP+jWncv3GlZc6cMVoe9u41ZpJetQri4s6+n5iVyMyNM5mzdQ45RTkAXBh3IQ9c9AAajULa1UXdaa3j7Z4fB1rZvTYD062LEPVSZibMmGE8nzWLCp2jTyechpfh1IlTxN1kfAj/5S/G6Hfffw8vvGDMvyNEfSOd1YRwBS/NRO1IYaFxV2v3bmOY1oSEssHDqiOraPdGO17d8Co5RTmMaj+KH2/5ka1Tt3Jr71sxKe//DEIIUR+89BKkp8PFF8P48RXfz9uTB8sha31Whe2UgrffhiNHPFRZIVxIvikI4QoeSmHKWJXB9vjtpH7ueCBxiwVuvdUIGpo3h+++M5rH84vPDh03qNUgmoQ0YdL5k9hy1xZW3rySS9tdipLefEII4bS0NHj9deP5yy87Tl3SFqNLj/Ir+2avXnDLLUb/ieeec3dNhXA9CSCEcAUPpTClLEwhc20mObtyHL7/xBPw6acQHg4rVoBfkxPc+829dHyzI9mF2QCEBYZx8MGD/G/S/+jboq9b6yuEEA3V7NmQnw+XXw4DBjguo83WMQEcfNv6+9+Ne04LF8LJk+6rpxDuIAGEEK7goRaIjm90pMtHXYi7Na7Ce/PmwWuvQUAA/HdxFvNOPU7HWR15d+u7JGUnsfqP1aVlGwdV3n9CCCFE1QoK4K23jOfTplVR0JrdWr4FAoyJ5q65xmiFePNN19dRCHeSAEIIV/BQHwj/xv40n9KckHZlp4/etcuYGA5g/KPfc/uuNry28TUKzYVc1/06dt+3m6u6XuXWugkhxLli/nxITYU+faCqibcrS2Gysc0X8e67kOO4YVkInyQBhBCu4IEWCNuFqLysLLj2WqMpveXFP/B5yGWcKTjDiHYj2Dp1K59e+ynnR5/vtnoJIcS5ROuzLQaPP171sK1VpTABDBwIgwcbI+ctWODaegrhThJACOEKbu4DUZRWxIaWGzj48EHs59nSGu64U3PgAPTsCbPeMtMzticrJq/gh5t/oE/zPm6pjxBCnKs2bzZafaOijJs3VaoihcnmvvuMxw8+cE39hPAEmQdCCFdwcwpT6v9SKTpVRP7B/NLRkpJzkrnq8ZVsXHIz4eHGDKedOo1hQo9RMmu0EEK4ie2L/q23QmBg1WWrS2ECox9EZKQxGd2OHdC7t6tqKoT7SAuEEK7g5hSm5AXJAMROjqXYXMwbG9+g44zRbPzoGgBeeesMnTsbk8xJ8CCEEO6RkwOffGI8v+OO6stXl8IEEBICN91kPJdWCFFfSAAhhCu4MYUp/0g+WRuyMIWZ2Nt7LxfOuZBHVjxKzpLXoTiMcVdnM/WWyOp3JIQQok4++8wIIgYPhm7dnNjAiRQmgDvvNB4XLDBGeBLC10kAIYQruDGFKXmh0fqwv89+hn82nD2pe2i292k4eikxMZqP54a7/JhCCCEqWrjQeJwyxbnyzqQwgTGx3IUXGp2pv/uuDhUUwkMkgBDCFdyUwqS1Lg0gDg87TIh/CI91fZOCFf8E4J13FFFRLj2kEEIIB06dgh9/NObaqbbztJUzKUw2N95oPNpSpITwZRJACOEKbgggDqQfYNv328jfn09ATACPP/E4u+/dw465D5Cbq7j+eqPznRBCCPf73/+Mj/rLLoMmTZzbptXDrWAuxN1WcfLP8q6/3nj8+mvIzq5DRYXwAAkghHAFF/aBKDIX8cJPL3DBOxew7OVlAMTcEEN042g2rWzHjz8awwfKzKVCCOE5ixYZj7aWAmcENQ+CThAUF1Rt2datYehQow/EV1/VspJCeIgEEEK4gov6QGxK3ETfuX15es3TFBcVM3DbQMAYfSk3F6ZNM8r9+98QHV2nQwkhhHDS0aOwcSOEhsL48e47ji04WbzYfccQwhUkgBDCFeqYwlRsLuaZH59h0AeD2J2ymw5NOrCyw0pCzoQQ0imE8IvC+fe/ITER+vaF225zYd2FEEJU6csvjcdx4yAszPntTi04Bf+B0z+edqr8tdcaM1v/8IOkMQnfJgGEEK5QhxQmrTVjFozh+XXPo7XmsYGPseveXbRY3QKA2JtiOXpU8fLLRvlZs9w23YQQQggHbClFV11Vs+2y1mfBCsjbl+dU+ZgYY4jYoiL4/vsaVlIID5KvIUK4Qh1SmJRS3NLrFtpGtmXNrWt4dcyrBBYHkrY0DYCYv8Tw+ONQWAiTJxsXFyGEEJ6Rng7r1oG/P1x+ec22jb0pFqZBZLzzc/VMmGA8Sj8I4cskgBDCFWqYwnT0zFG+2PdF6etbe93K7nt3M6ztMGM3ISYuWHkB7f/dno1/hrJ0qdFs/tJLLq+5EEKIKnz7rXGPKD4eIms4Z2fE4AgYB2Hdnc97sgUQ33wDxcU1O54QniIBhBCu4GQKk9aaD7Z9QM93ejL5i8n8nvY7YLRChAWevcAopYgYFMF5/9eav/7VWPfUU9CypVtqL4QQohK2lgDbF3t369zZmOX6zBmj5UMIXyQBhBCu4EQLxKmcU4xfPJ47v76TnKIcLu90Oc1CmlW522XLYMsWiI2FRx5xZYWFEEJUp6DgbF+E2oy+dHr1afgK8vY71wfCRtKYhK+TAEIIV6imD8Tnez+nx+wefHPgGyKCIph/9Xw+m/QZ0WEVx2JNfCeRHSN2kL4yg2eeMdb97W81G/lDCCFE3a1eDbm50KePMU9DTZ36+BS8Dlmbsmq0nS2A+PJL0LrmxxXC3fy9XQEhGoQqWiBeXPci03+cDsDI9iP5cPyHnBdxXqW7SlmcQuZPmRw9vzm7dsF558Hdd7ul1kIIIapgG761tulL2mx8+1cmVaPt+veHuDj480/YuRN6967d8YVwF2mBEMIVqugDMbHbRJqFNOPNsW/y/U3JY1fxAAAgAElEQVTfVxk8APT4sgcd3+3MM99FAfDMMxBU/SSmQgghXMhiga+/Np7XdPhWG1sAQQ1H+DaZ4MorjeeSxiR8kQQQQriCXQpTXnEec7fORVvbnbtEdeHYI8d4oP8DmFT1/+UCmgSwMqAFuw/50aEDTJnixnoL4QZKqWFKqW1Kqd+UUluUUgMdlIlVSi1RSm1XSm1SSv2slBrqjfoK4civv0JyMrRtCz171nIn1ntLyq9mLRBwNmixtYII4UvqnMKklBoGzLTuqwh4QGu9sVyZvsAPwAG71Uu01q/W9fhC+ARrC8SvgancMudCDqQfIMAUwG0XGlNG24+wVBmtNdqsMWsTM2YY6/75TwgIcFuthXA5pVQk8AVwhdZ6g1IqHvhKKdVOa23fk/RFIA24TmutlVLXAJ8CMtaY8An2oy+pmn//B2qfwgQwfLjR923HDjh2DNq0qV0dhHCHOrVA2F0o7tdaXwBMw7hQhJYrGgV8rrUeaLdI8CAajCJzEc9cCoObLuVA+gG6R3end1zNklazt2SzocUGll9/lGPHoEsXuOEGN1VYCPcZA+zXWm8A0FonAEnAiHLlEoFIwJagF21dJ4RPWL7ceLSlEtVGbVOYAIKDYcwY4/mKFbWvgxDuUNcWiAoXCqWU7ULxtV25KGC0UupX6+tVwL+11tl1PL4QXrcnZQ83Z77K9mGgtGbaoGk8N/w5gv2Da7Sf5IXJFKcWs22tMXPQ//1ftdNKCOGL2gOHy607bF1v7x/AXCBFKXUGSAXGOdqhUmoqMBUgNjaWhIQEV9a31nJycnymLr6ioZyT9PRAdu0aTHCwGbP5ZxISajkUUqrxsGfPHiNcrqH27ZsDXViwII2uXXfXrg4+qKH8nbhSfTsndQ0gnL1QfAF8Ym2mbgq8BcwDri6/Q1+8UNS3X6onyDkx7Dyzkyd+e4JiXUzb0/Dy6WFExY9j488bq9/YnhnjfwSwOCOWqKhCWrXaWPuLlo+Qv5OKzoFzojD+ou2VULHF+ymMdKXztNaZSqm7gG+UUgO01mW211rPxQg26Nevn46Pj3dLxWsqISEBX6mLr2go5+S//zUehw/3Y/ToYbXez2+Rv5FBBj1796RZfNXz/jjSoQO88grs3BnF4MHxBAbWuio+paH8nbhSfTsndQ0gnLpQaK3z7Z5nKKUeB44rpULs37O+73MXivr2S/UEOSeGgSUDeT/pfQZmhPLaO5sIf7Af1OK8ZHyfwW+nfyMtJIQD+eG88qSq00XLV8jfSUXnwDk5AYwst6418Fm5dX8BHtdaZwJord9TSs0AegHb3F5LIaqwcqXxaEshqq26pDCBMYx39+6wZw+sX1+ry4sQblHXUZhOYFwY7LW2rq+KH1AAFNbx+EJ4lNaaeTvncTr/NADB/sFsuGMD7wVMJLyIKmeirkrygmQAluXHEhmpmDrVVTUWwuO+Ai5QSvUEUEr1B7oCPyqlflFKdbKWOwBco5QxNJlS6hKgMfCnF+osRCmLxXUBROkoTLXoRG1z2WXGo/SDEL6krgGEUxcKpdQN1g7XKKX8gX8B87XWljoeXwiPOZVzivGLx3Prl7fy4IoHS9c3CmxU7UzUVTHnmkldaiTKriaW+++H8HCXVFkIj7O2KEwCPlRKbQJeBy4HQoE2QIS16H0YAcM2a/+4l4GJWus0z9daiLO2b4e0NGPm6c6d67av0lGYajGMq83Yscbjd9/VrS5CuFKdUpiseau2C4XGSF9ydKEIAVYrpSyABtYCf6/LsYXwpM/3fs7d39xNen46EUERXNbxsrIFqpiJujppX6VhybWwm8ZkBIfw0EMuqLAXZWVlkZKSQnFxMREREezbt8/bVfIpnjonYWFhtGrVClMtW8XqQmu9BrjIwVut7MqcAm70WKWEcJJ960Nth2+10Za6pTABDB0KoaHw229w8iS0aFG3OgnhCnWeB8LJC8VHwEd1PZYQnpaRn8EDyx9g0e5FAIxsP5KPJnxEq8atyhasYibq6iQvNNKXVhPLbbdBTEydquxVWVlZJCcn07JlS0JCQsjJySFcmlPKyM7Odvs5sVgsJCYmkpaWRkx9/oMSwgu+/954HD267vtq/URrdvXbRWjn8qPbOy8oyJgT4ptvjLrddlvd6yVEXclM1EJU4nT+aXrM7sGi3YsIDQjlrbFv8f1N31cMHqDWKUxFKUVkfJ9BCYo1RPPwwy6ouBelpKTQsmVLQkNDUXW9dSdqzWQyERsbS2ZmprerIkS9kp1tdFY2mWBE+ZlLaqHZuGZwBQS1CKq+cBVs/SAkjUn4ijq3QAjRUDUJacL4LuPZk7qHjyZ8RMemHSsvXMsUppT/pYAZNtOEgWMC6dKlDhX2AcXFxYSEhHi7GgIICAigpKTE29UQol5JSIDiYhg0CJo08XZtzrIFED/8ACUl4C/f3oSXyZ+gEHZWHVlFWEAYg84bBMDrl71OgCkAP1M1qUm1TGFKnm+kL/1ALI8/WE3hekJaHnyD/B6EqDlXpi+B9SbRBig6v4jAmNpP4tChA3TqBAcPwqZNMHiwa+onRG1JCpMQQE5RDvd/ez+j5o/i5qU3k1uUCxjDtFYbPECtUpjyD+eTvSmbPPw41S6qdKQNUX9NnjyZJUuWsHjxYnr37u3t6gghashlw7da/fnvP+F1KDxe91HrJY1J+BIJIMQ5b80fa+j9bm9mb5lNgCmA23rfRpB/DfNVa5HCFNwumI/79GYWHZn6kF9tp5AQbrB48WLi4uIcvvfxxx+jlCqzfGe9oicmJkq/AyHqqT/+MO7wR0TARY6GhqmF6EnRcCUERAfUeV8SQAhfIilM4px1puAMT6x8gve3vw/ABbEXMO+qefSK61XzndUihWnvPsV/t0USFhbJpzKqhk85fPgwsbGxlb7fvXt3du/eXfr6zJkzfPfdd2RkZHiiekIIN7ClL40c6bo+Bm2easMfCX8Q3Dq4zvsaNgwCA2HLFkhPh2bNXFBBIWpJ7nmKc5LWmkv/eynvb3+fQL9AZsTPYPNdm2sXPECNWyC01rz1lvH8lluMO17CdyxZsoTdu3ezceNGh+/v2bMHf39//P39iYuLIyUlhQULFpCcnOzhmgohXMXV6UuuFhZmzAmhNaxe7e3aiHOdBBDinKSU4skhTzL4vMFsv3s7zwx7hkC/2ndwq2kfiH0PHqbT3J10JYsHHqj9YYXrvfrqq6SlpTFz5kxuvPFGTpw4UaFM9+7dKSkpoaSkhFOnTtG5c2cWLFhAt27dvFBjIURdFRef/VLuqg7UANk7smEfmAvMLtmfrW62YEcIb5EUJnFOsGgLH+/4mIz8DKYNngbAdd2vY1L3SZiUC+LoGrRAaK1J/CSVPpZCevdpz/nn1/3wPs1bowFpXaPiRUVFvPjii7z++ussX76cwYMHc+zYMYYNG8aiRYvo37+/3a41+fn5pKWlkZSUxI4dO+jVq5atV0IIr9u0CbKyoEsXaNPGdfvdc+0eOAyFIwsJ7VT7yeRsRo+GJ580AgitvffxKoQEEKLB23lqJ/cvv59fjv9CgCmAid0m0q5JO6MDLC769K1BHwilFDNa9yP4dDpTHm3kmuOLOpkzZw4zZswgOjqan376iQsuuAAwWiNatmzJ8OHDGT9+PM899xwAe/fuJSwsjLCwMFq0aEHXrl3p27evN38EIUQduHr41lLWS4Pyc821plcviImB48fh999BGj2Ft0gAIRqszIJM/r7m77y1+S0s2kJsWCyvjXmNtpFtXX+wGqQwbdsGP+8MoEmTOL6+1vVV8Tk1bAnwhh49ejBz5kwmTpyIX7kg8LHHHmPixIksWLCA5s2bM2XKFKZMmeJwPytWrCAgwBhtZcKECe6uthDCRWwBhKv7P2iz9fPPRQnjJhOMGgULFxqtEBJACG+RPhCiwdFas+C3BXR5qwuzNs0C4KH+D7H/gf38pedf3DPBlpMpTJZiC+/PMcreeisE131gDuECQ4YM4brrrqsQPNi0adOG6dOnExp6NgXh008/5ZFHHilTLiQkBH9/f7799ltuvvlmt9ZZCOEaGRmweTMEBEB8vGv3bQsgXNUCAdIPQvgGaYEQDdInuz4hOTeZwecNZvbls2s/upKznExhOj4/lVHvHSKJNtx1Vyv31kk4xWw2o2vQSuJvHd/x+PHjvPHGG7xlG07Ljtaanj17uqyOQgj3WbXKaCgdOtQY6cilXJzCBEYLBEBCAhQWQlANpy0SwhWkBUI0CCm5KRzKOAQYfQzeHPsmH034iHW3rXN/8ABOpzDtnplME11Mu/aq4XeeridGjBhBQECA08vvv/9eZlvbaEz2ywcffODFn0gIURPuHL7V1SlMAM2bwwUXQF4e/PKL6/YrRE1IC4So1wpKCpj16yxeWPcCvWJ7sXbKWpRSdGjagQ5NO3iuIk6kMBWlFBG8O4MSFH0ei/ZQxUR1Vq9eXasWCDBaL3JyciqUKSwsdEndhBDupbUbO1DjnhQmMOr6229G8DN8uEt3LYRTJIAQ9ZLWmiV7l/DXVX/l6JmjADQKbER2UTaNgxp7vkJOpDDteC0FP2Czf1Puv60Oc04Il6qs34MzEhISCA8Pd/ieDOsqhO/btw9OnDBGNnLLf1k3pDCBEUC88ooRQPz73y7dtRBOkRQmUa9orfnu0Hdc9N5FXP/Z9Rw9c5QeMT34bvJ3LJ+83DvBAzjVApH4X2OWYvOlMYTWfThw4WXTpk1Da13psmPHDm9XUQhRDVv60ujRTs8DWiPuSGECuPhiYxCO7dshJcW1+xbCGRJAiHrlTMEZJi2ZxNakrTRv1Jw5V8xh+93bGdPRDcmrNVFNH4jsfXk0OZVNHn5c+nSUBysmhBCiMu5MXwL3pTAFB8OwYcbzH35w6a6FcIoEEMLnbU7cTEFJAQBNQpowI34G/xn5Hw49dIipfafib/KBTLxqWiA2vmjcItrZOIr+F9c+ZUYIIYRrFBTA2rXGc9vIRi7nphQmkOFchXdJACF8ktaatUfXMnr+aPq/35+Ptn9U+t6jgx7liSFPEBrgQ3lAVfSB0FqT96WRvhR+dSzumIZCCCFEzfz8M+TnG30f4uLccwx3pTBB2QCiHszXKRoYH7h1K8RZWmu+P/w9z//0PL8cN8anaxTYqLQFwmdVkcKUnJBNRE4+6QQy7pkmHq6YEEIIR9w1+7S9Fne3IPFYIsrf9XeOuneHFi3g5EnYvRtk6hnhSdICIXzGioMr6DO3D2MXjuWX47/QJLgJzw57lmOPHOPRQY96u3pVqyKFadOLRuvD/pYxtOsgzQ9CCOEL3Dn/g02nWZ3gUTD5u/7rllJnWyFswZAQniIBhPAZGfkZ7Di1g5iwGF4a+RLHHjnGP+L/QdOQpt6uWvUqSWGyFFvwW2v0fzjvtlhP10oIIYQDSUnGPAqhoTBkiLdrU3vSD0J4i6QwCa/YcnILc7bMoVFgI2ZeNhOASd0nYdEWJnWfRLB/sJdrWEOVpDAd+t8ZwoqL+VOFcuVjjbxQMSE8Tyk1DJiJcY0pAh7QWm90UC4KeBfoChQCK7XWT3myruLcZPvCHR8PQUHuOYa2aLI2ZcF+IN49xxg50miJ+Oknoz9HSIh7jiNEeRJACI/JLszmk12fMHfbXLYlbQMgNCCUGZfOIDwonEC/QG7udbOXa1lLlaQwLf2zCYvozaghZm5pIulLouFTSkUCXwBXaK03KKXiga+UUu201nl25YKAb4AntNbrrOuaeaPO4txjP/+Du+hizfZB241vWne75xjR0dCnD2zdagQR7kzHEsKepDAJtzty+ghTv55Ki9dacM+397AtaRtNQ5ry6MBH2TZ1G+FBjmfyrVccpDBpDfMWKHYSySX/J9+LGjptNwxKfHw8r7zyiluP54lj1NIYYL/WegOA1joBSAJGlCt3C7AReEgptUUpNR8I8GRFxbnJYvFM/weA8P7h0MW9x5A0JuEN0gIh3CK3KJewwDAAisxFvLftPQCGth7K3X3v5trzr61/aUpVcdACsW2LZu9eRVQUXHaZl+olnNavXz+2bt1a6fubN2+mX79+/Oc//+Fvf/tbmffMZjMvvvgiTz3lXPbN4sWLeeSRRzh16pTD9z/++GNuu+22CuuHDBnCzz//7NQxvKg9cLjcusPW9fYuAXoDVwJ/AjOAhVQMNFBKTQWmAsTGxpKQkODaGtdSTk6Oz9TFV9SHc3LgQCPS0voRG1tAUtJGKvlv6Bovuf+cxMREAr1ZujSHK6/c4rbjuFJ9+DvxtPp2TiSAEC5z7MwxPt/3OUv2LqGwpJBtdxtpSl2jujJzzExGtR9F95juXq6lmzjoA3Hw+t38G03mZR0J8KU5K4RDCQkJlJSUOHyvSZOzw+8+/vjjPPTQQwAopfDz8yM6OpouXZy/zXj48GFiY6vuVN+hQwcOHTrk9D59iALM5daVULHFOwb4WGt9FEAp9RKQqZRqpLXOsS+otZ4LzAXo16+fjo+Pd0O1ay4hIQFfqYuvqA/nZMMG4/HKK4O59NJ4tx/P3edk8GB4+mn4449GdO4cT4sWbjuUy9SHvxNPq2/nRAIIUSeHMg7x+d7P+WzfZ2w5efbOR1hAGEnZSTQPbw7AIwMf8VYVPaNcC0RBRglN/zhNFJrGt8h/s/qgUSPnOrn7+fnhZ5eqtnHjRnJzcxk2bJjTx1qyZAm7d+9m48aNDBw40GGZI0eOEBxctpXu6aef5umnn3b6OF5yAhhZbl1r4LNy61KALLvXFrtFCLfxxPwPYKQ16hLt9r/owEC49FL45hsjjWnKFPceTwiQAELUweojqxk5/+z3hLCAMMZ1HsfEbhMZ12lcaQrTOaFcH4gfN/pzIwMZ1TKbJSMDvVgx36D+WXkH8jlXzGFq36kAzN06l7u/qby3of7H2X4Gfef2Le2MX1U5Z/Xr14/t27ejHEwV7ufn53A9wKxZs7jmmmto1sy5fi5vvvkmaWlpzJw5kxtvvJF169bRqlWrCuXat29fX1sgvgJeU0r11FrvUkr1xxhl6Uel1C/AFK31QYyO1n9TSi3WWmcDjwA/2ne0FsLVsrNh/XrjXs+ICslyrlWcXsz66PXQGMh077FGj5YAQniWBBCiSlprDmUcIuFoAiuPrCQiKIL3x78PwKDzBhHXKI6R7UcysdtExnQYQ0jAOTqGXLkUpvnzIYtALry3GZV87xQ+6I033uCBBx5wuvyvv/7Kp59+yvLly6stW1RUxIsvvsjrr7/O8uXLGTx4MMeOHWPYsGEsWrSI/v37V7ptfn4+mZmZJCcn89tvv9GuXTun6+hpWutMpdQk4EOllMZIX7ocCAXaABHWckuVUh2BzUqpfOAYMMU7tRbnitWrobgYBg0Cu8xE97Al8nlguBpbR+offjDuZzmY01QIl5IAQlSQmJXIysMrWXN0DWuOruFE1onS9yKCInj3incBYwjWxMcSMSn5pLJPYTqdZGbZUhOguOkmr9bKZzjbIjC179TS1ojqbJ1aeYfn2rJYLJX2gzCZTJjsrsqpqanceOONdOzYkSeffJJhw4ZVSDmymTNnDjNmzCA6OpoVK1YwaNAgAF599VVatmzJ8OHDGT9+PM899xwdOnQAjH4StlaPkJAQwsPDiYuLo2fPntx9t5vGhHQRrfUa4CIHb7UqV+5l4GWPVEoIYMUK43HsWPcfS5utn3seuER27gxt2sCxY7BjhzG0qxDuJN/8znFF5iI2J27myOkjpes+3fMpty+7nfm/zedE1gmahTRjYreJvH352+y4Zwf+prNxpwQPVnYpTAn3/Ml/CzdwX7cU2rTxbrVEzTz88MMEBAQ4XG6//fbScgUFBUyYMIG+ffuyc+dOlFLccccdlQYfPXr0YObMmWzdupUePXqUee+xxx5jz549dO/enebNjT5DU6ZMMfKnrUteXh7Jycns3LmTBQsWcPHFF9OzZ0+HqU9CCMe09nAAYfFcAKHU2VYIWx8PIdxJWiDOIWaLmUMZh9iWtI1fE3/l18Rf2Z60nUJzIdMvns7zw58HYFT7UUzoMoFL217Kpe0upUdMDwkUqmNNYdJKoVcmE0URQ8bKf6/6ZMuWs4MAvP7667z//vvs3r27Qrnk5GQmTpyIyWRi3rx5BAcH87///Y9LLrmEq666iiVLllTYZsiQIVUeu02bNkyfPr3C+vz8fMzm8gMaGf71r38R5K4pdIVogPbuhePHISbGQ3foPZjCBEYA8d57Rj8IJ0eUFqLW5BtOA3Wm4Ay7kncxtPXQ0jSIgR8MLDNSkk3XqK5EBkeWvu4Z25Mvb/jSY3VtEKwtEAfX+dO0IIt0Arl8ursTbIWn5eXlMXDgQAYNGsQHH3xASIjR56djx4789NNPvPbaa/j7l/1YNZvNZSaZKykpqbSlAiiz/aBBg9i5c2elZV9++WWmTZtW2x9HiHOKrfVhzBjP9BHwZAsEGJ3CTSb45RfIyQEnB5YTolYkgKjnMgsy+fnPn9mfvp/9afuNx/T9nMoxZsY5/uhxWjU20hy6RnUlOSeZXnG9GNByAANaDuCilheVCR5ELVkDiK3/NdEcON45hsim0nu6PoiPj2ft2rUO33M08tKGDRscDr3asWNHZs+eXWH9iBEjKt2/I/v27aNr164A7Nixo9Jy9Wm8cCF8gSfTl8CzfSDA6BTevz9s3AgJCXDFFZ45rjg3SQDhwyzaQlJ2EkfPHC2zXNj8Qu676D4A9qXt44pFFT8lgv2D6RHTg4z8jNIA4qMJH5XpvyBcyGLBjB+N9hjzX3W5v+pJwoTvWLlyJRaL8wO11zRtaPXq1WVaILKzswkPD6+0fPkWDCFE3WVnw7p1xh16W18Bt/NwChPAZZcZAcTy5RJACPeSK5UXaK3JyM8gKSeJpOwkTmafJCkniYcHPFw6DOqNn9/I53s/p9hSXGH78XnjSwOILs26MKLdCLo060KXqC6lj60jWlfotyDBgxuZzWzmBsItJZzwC+W6e6TtuL4IDHTvPB32k86BESBIkCCEZ/34ozF868CB4OSULXXm6RQmgHHj4NlnjTkh3n4bGUZcuI1cxVzgZPZJTuefJrMwk4z8DNLz0knLSyMtL42BrQYyoesEANYdW8e1S64lPS8ds67YMXLS+ZPo0NQYwlGhKLYUExMWQ9vItsYSYTz2jO1Zuk2TkCasumWVZ35QUTmLhYNM5Dwgs38sgYHyqX0uS0hIcPsx/vWvfxEbKy1dQjjD0+lLYJfC5MHLQZ8+EBtrdBbfvRt69qx+GyFqo84BhFJqGDDTuq8i4AGt9cZyZRQwA7gOo1FvG3C31jq3rsevCa01f2b+SV5xXpkltziXvOI8+rfsT+dmnQHYcHwDS/YuIbswm4MnDhJ8IpjMwkyyCrMoLCnk0ENnZ4gdNX8Ue1P3Ojzmff3uKw0ggv2DSclNAaBxUGNahLegeaPmNA9vTotGLcpMwjZr7Czeu/K9c2s253osMz+OWJoCmouejPF2dcQ5wDaXhBCiap4evrWU7T6hX5WlXMpkMlohPvzQaIWQAEK4S50CCKVUJPAFcIXWeoNSKh74SinVTmudZ1f0VoyZSHtrrfOVUh8BLwFVTvl6PPM4935zL0XmIgrNhRSZi0qXmWNm0iWqCwDPrX2ORbsXVShTWFJIj5gebLprEwAaTds32lZ6vHfHvVsaQOxJ3cPMjTPPvplatmyRuYhAPyP1oWPTjmitaRzUmKYhTWkW2oyokCiiQqPo3/Ls7LK94npx8rGTNAttVrptZaJCo6p8X/gOS5GFLYkPEIhmY6No/nrlOTobtxBC+KB9++DPPyE6Gvr29dxxvZHCBGcDiG+/leFchfvUtQViDLBfa70BQGudoJRKAkYAX9uVux6Yo7XOt75+A1hNNQFEoyONGHHjCIfvJYUmke6XDkC/wn50K+pGQvcE3h77NgCtU1vz6n9fJb1FOtxlbGNSJj574zP8zH4opVAo49H6PHR2KOv91gPQydKJ70u+p6RdCRuf2MigCwfROKgx5iFmVInC9KSp9K7C8588T/aW7Ep/jvWsL/P6D/5g4NGBmAKNT5Vd43eRvSWbHl/1oPFFjQE4Mv0Ipz46VdXpqcDR9u2eb0fz243JqVI+TeHQo4eq2kUFjraPvi4arjLez92Xy84RlQ8z6Uj0ddF0er1Tme1Du4bS+8fepWU2tNmALnZu9mKg0u0dneeacOb3dHLOSfxKmnGSYExToiXnVAghfIinh2+1MYWYCO8fTnZEza47dTVqFAQEwIYNkJ7uuT4f4txS1wCiPXC43LrD1vVVlTsMNFVKRWitM+0LKqWmAlMBOtOZqJxK7sTnQBFFAIRY/40NH8slAy7B3+RP0NEgwnLCaFbcrEw+crPsZlD5EOyl+wQIJJDAxoH0D+1PSGIIxRTDKaAEfv7pZwiwFjwCJFW+T0d+WvtThe23/boNbEld+2q+T0fb79+xn/0J+41122u+T0fbJ/6eSE5OjnFea/GzJ/6eSGJCovHCun1RcFHZvPGTVPl7Kq+y7R2d55pw6veUDln48wLdeKHLWhISGtfsIA1IREQE2dlnL5Zms7nMa+HZc1JQUOCR/hhC+LKvrbczx43z7HHDuobR99e+Hv8/GB4Ow4bBqlXw3XcwebJHDy/OEXUNIBRns/xsSqjYYFe+nO2rYYV7AVrrucBcgL69+upBK5zP8zWFmAhoYnzbsxRbKL6iGOWvCIw5my5UeKzQ6f0BKH/F+r3rS8dct20f2DywdIz4ojVFNbpbXtn2AVEBpXe7Sy4swfyW4xloK+Noe7/Gfvg3Mn7N5v5mSh6qwbdycLi9KcTELzt/IT4+HssQC8WXVRwpqiplfk/W7V3xe3K0vSd+TwuGmLlnmj+D+ImrxveG1q1rdIyGZN++fWWGKK1uyNJzkSfPSXBwMBdeeKFHjiWEL0pPN4Zv9ff3cP8HL7viCiOA+PZbCZi30dUAACAASURBVCCEe9Q1gDgBjCy3rjXwmYNyrcuVyQHOVLVzFaAIalGzMddtTAEmh9vWan92/aMdbR8YXbdhIB1t7x/hj39E7X89jrb3C/XDL7T2vbkcbV/ZeXaWS39P1Wzvrt/Tx1/7UwjczHwwzajTMYQQQrjO8uXGPJ/Dh0NEhGePrS3aGInJ+WlmXGbcOHjkESN9q6TECKCEcKW6ZgN+BVyglOoJoJTqD3QFflRK/aKU6mQtNx+4Uyll+wb2IPCFtp9dSYh66OhRWLsWgsnnWj4DPw8OtyGEEKJKX31lPE6Y4PljZ67L5KfAn+BRzx+7Y0fo0gXOnDH6QgjhanUKIKz9FyYBHyqlNgGvY4y2FAq0AWzx/jxgHbBJKbUFaIwRRAhRry1caDxeFfQdjcn2bA894RIXXXQRcXFxxMXF0bRpU5RSpa/j4uJ4+OGH67T/o0ePopTi999/d1GNhRDOKCgw+gAAXHml54+vtTYGW/HSfSVbn49vvvHO8UXDVudGLa31GuAiB2+1sitjBqZbFyEaBK1h/nzj+c2Bn0IhEkDUQ5s3by59vmrVKkaNGsWpUzUbAU0I4XsSEiA3F3r3hjZtPH/8JvFNiC+J99pABldcAa+9ZnQif+klr1RBNGDybUeIWtqyBfbvh5gYGO232lgpAYQQQvgEW/rS+PHerYe3DB0KTZoY82BIA6hwNfm2I0Qt2VofbrwR/LV1JCrpA9Hg5Ofn89RTT9GtWzfi4uJo1aoV06efbUz9+OOPiYuLY8mSJfTs2ZPo6GgGDRrE3r1lZ6ffuXMnF198Me3ataNz584sXbrU0z+KEOcMrWHZMuO5N/o/+IKAgLPBk3zcCFeTAEKIWiguhkWLjOe33AKYrUPuSgtEBUp5Z3GVzMxMzGYzq1at4v/bu+/wqKr0gePfk14IIaQBIZQgAtKkKqIYBEGxrKIgKFIUUVd0LeDqyq6o609xFV07IoioqKAINtBFiBSpQQSkSREChE4S0icz5/fHmZl00jMzyft5nvvMnTu3nLm5mXvfU48dO8a6det48803WVugZeKpU6eYO3cuCQkJnDhxgrZt2/Lgg4WbeT377LO8++67HDhwgL///e+MHTuW9PT06kuoEMIpMRGOHoWYGHBVT8YpP6ewsctGeMM1xwcYOtS8LlzoujSIukmedoSohKVL4dQpuOgi+83JZu+nTwKIOqdJkyZMmzaN1NRUFi5cyIIFCwgICGDv3sKjus+dO5fw8HCUUtx6662F2lYATJ06lY4dOwIwYsQI0tLS2LNnT619DyHqE0fpw403Vm+GQkXkpeaRsS3DDEDrIldfDcHBpsrtoUOuS4eoe+RpR4hKcDaevtN+c3IEEFKFqRitXTNVly1bttChQwf+/ve/s2PHDsLCwoiKisJqzR/oMSYmhrCwMOf7oKCgYqNNd+rUyTkfHBwMQFpaWvUlVAjh5A7Vl7TV/kPkwietwMD8AfSkGpOoThJACFFBKSnm5qRUgRE+pQpTnfXwww8zZMgQvvnmG6ZMmcLYsWOxWAqPvq7KkcVZnnWEEFW3dy/89huEhEB8vOvS4Q4BBEg1JlEz5GlHiAqaPx9ycsyNKTbWvlCqMNVZOTk5nD59GqvVSnZ2Nk888USx6ktCCPexYIF5vekm8Pd3YUIchZQuvi1cdx34+cGqVXD8uGvTIuoOedoRooLmzDGvY8cWWCgBRJ312muvsW3bNiIjI7nooosIDw+nR48erk6WEKIU8+eb1+HDXZsObXOPEoiGDWHgwMI9UwlRVVUeSE6I+mTPHli7Fho0gFtusS8sWOleAgiPNnDgQDN6bAGXXHIJmzdvLrRs8uTJzvmxY8cytlA0WXg/rVq1KrZPoMRldYVS6krgVcw9JheYqLVed571nwamAG211n/WSiJFnbRnD2zZAqGhpgGxK7lLFSYw1Zi+/960g7jnHlenRtQFEkAIUQFz55rXW281PVsAztIHrZTUcxf1nlKqEbAQuF5rvVYpFQ8sVkq11lpnlrD+DUBT4EjtplTURW5TfQncpgoTmN6ovLxg2TJITTUBlhBV4QaXtRCewWbLDyDGjCnyAbiur0Ah3MtgYLfWei2A1joBSAYGFF1RKdUOeBj4W20mUNRd7lJ9CdyrBCIyEvr1M2MYOUboFqIqpARCiHJasQKSkqBVK/ND7OQogfD2RkIIIYgD9hVZts++3Ekp1RCYBdyhtc45X+mdUmoCMAEgOjqahISE6kxvpaWnp7tNWtyFK8/JoUNBbN3amwYNLPj6/kJCgourCe40LxarxS2uk27dmpKQ0I433zxDixZbXZoW+d8pztPOiQQQQpSTo/H06NFFmjrYu3DVUgIhBIAiv/KGQx4F8mGViRY+BJ7TWh8sa4da6/eA9wB69uyp413ZN2cBCQkJuEta3IUrz8lzz5nXYcN8ufrqK12ShoKO7j7KHvbg6+/rFtdJ587wxhuQmNiYDh3iiY52XVrkf6c4TzsnblCwJoT7S0uDL78086NHF/lQemASoqDDQIsiy1rYlzuEABcDzyil1iml1mHaQXyllBpbK6kUdY47VV8C96rCBBAebgaVs9ng889dnRrh6dzkshbCvX3xBWRlwRVXQJs2RT50VGGSAEIIgMVAF6VUZwClVG+gPbBcKbVGKdVWa52mtW6ttb7UMWHaSdystZ7juqQLT7VjB2zfDmFhMKBYaxsX8rZPbsIx+Oknn7g2HcLzyROPEOXw4YfmtUhvnYZjFGqpwiQEWutUYBgwWym1AXgNGAIEAS0B6f9FVLtPPzWvN98Mvr6uTYtDzF9jiM+Lh4dcnZJ8N9xguiHfsMGM2C1EZUkAIUQZ9u+HlSshMNB031qMlEAIUYjWeoXWupfWurfW+jKt9VqtdZLWurnWelMp27SSMSBEZdhs+Zk8d97p2rS4u6AgE2QBzJvn2rQIzyZPPEKUwdF16y23mBE9i5FuXIUQwmWWLzc95LVuXaSHPFGigtWY6vB4lqKGSQAhxHmUOvZD0ZUw3bgKIYSoXY4e8saMca++LJJnJ7Ox60b4wtUpKWzAAIiKMqN2b97s6tQIT+VG/2pCuJ9Vq+DAAYiNhf79S1lJunH1ePHx8TRo0IAmTZoQExND+/btefbZZ8nJySlz2zlz5tCkSZNSP1dKsXTp0mLLR40axdgSG9UIIcorNRUWLjTzxXrIc7HcY7lkbM2As65OSWE+PjBihJmXxtSisiSAEOI83n/fvI4eDaUWMEg3rnXCxIkTOXbsGEeOHGHp0qUsWrSIRx55xNXJEkKcx4IFpoe8+HhThcmdNB3flJ5besLNrk5JcY5qTB99BOXIJxGiGHniEaIUZ86YmxPA3XefZ0VpRF3ntGrViscff5z5jo7lhRBuyVF9yR0L8/yi/GjQtQFEuDolxfXqBV26wKlTsHixq1MjPJE88QhRio8/NjkzgwaVkbMl3bjWSZmZmfj4+ADw888/069fP1q2bEmLFi247bbbOHz4cBl7EELUpD17YM0a0y1piT3kiVIpBRMmmPkZM1ybFuGZfFydACHckdbw3ntm3vEjWyopgShTgkqo0PoNujegZ2LPYtvH63jnsk09NpG+Ob3E7QuuV1E2m43Nmzfz4osvMm7cONatW8dTTz3F3LlziYuLw2KxMGXKFG6//XZWrlxZ6eMIIarG0XXrsGEQHOzatJTk5JcnOf3daWgLxLs6NcWNGgWTJ5terP74A9q2dXWKhCeRAEKIEqxbB7//bnqquOGGMlaWAKJOePPNN5kzZw7nzp3Dx8eHF198kfvuu49rr72Wbdu2cdlllznXtVqtWCwWrI7Sp0pSUmolRKVYLGUM8OkGzm06x7EPjsH5qsC6UGioaUz9wQcwcya89JKrUyQ8iQQQQpTAUfowbhz4+ZWxsjSiLlNVSgRK275gCUV1mDhxIi+++CJ//PEHnTp1omvXriil2L9/P0899RSPP/54pfbbsGFDMjIyii1PS0sjLi6uqskWol5atAiOHIH27eGKK1ydmpJpq32QBTfu4XvCBBNAfPABPPcc+Pu7OkXCU8gTjxBFpKTA55+b+fHjy7GBdONap7Rt25bx48dz//33Y7VaiYmJYePGjVXa3++//15omdaa33//nZYtW1Y1uULUS2+8YV4nTnTf5mfaZg8g3PhJ65JL8htTL1rk6tQIT+LGl7UQrjFvnukW8Kqr4IILyrGBjERd5zz99NMcOHCA119/nSeffJKFCxcyc+ZM8vLysNlsrF+/njmO7l/K8PDDD/P666+zbNkybDYb6enp/OMf/yAtLY2RI0fW7BcRog7assWM0RMS4n5jPxTiqOHoxreGgo2pHSXvQpSHBBBCFKB1fo8UZTaedpCRqOucqKgoJk2axNNPP02nTp1YtGgRs2bNomnTpsTExDBp0iRiYmKc6588eZImTZoUmoYPHw6YAeNeffVVJk+eTEREBB07dmTXrl2sXr36vAPQCSFK5ih9uOsuE0S4K2cVJjd/0ho1CgIDTWPq3btdnRrhKaQNhBAFrFoFW7eaxtM33VTOjaQbV4+XkJBQbNm//vUv/vWvfwHQrFkzbiilNf3YsWPLHFH6zjvv5M477wTg3LlzhLjzU48QbuzUKVNKDPDAA65NS1k8oQ0EmMbUo0aZhtTTp0u3rqJ83DwuFqJ2OXK27r23Ao3JpBcmIYSoFe+/D9nZcO21HtDtqL12qyc8aT36qHn98EM4fty1aRGewQMuayFqR1ISfPUV+PjAffdVYEMJIIQQosbl5cHbb5v5hx5ybVrKw1kC4QGF0+3bw403msFT33zT1akRnkCeeISwe+cdUxvpllugWbMKbChVmIQQosYtWmQyetq2hUGDXJ2asnlKFSYHR0/Vb70F6SWP0SmEkwQQQmCKxGfONPMVztmSEgghhKhRWsMLL5j5hx/2jGF3PKURtUPfvtCnD5w9C7Nnuzo1wt1JI2ohgM8+M43zunc3P6AVIgPJCSFEjfrxR9i82XRwMW6cq1OTz6ZtJJ9L5kDKAY6nH+d01mlOZ54mLSeNEZYRZiUFn23/jD2n9xDsG0xYYBjNQpoRExJDTMMYwgLC3GZU+smTYehQePVV+OtfTZVeIUoil4ao97SG11838w8+WImaSFICUYjW2m1uhvWZ1trVSRCi2jhKHx591HQ56gpHzx3FarMSGxoLwNK9S7nps5vIseaUuP7Q3KFmxhvm/z6fr3Z9VeJ6A+MG8r87/weY/9vE5ES6RHfBz9uv+r9EGW680VQR++MP+OILGDGi1pMgPIQEEKLeW7MGfv0VIiIq+WMpbSCcfH19ycrKIigoyNVJqfcsFgs+kn0o6oCffzZTaCjcf3/tHTcpNYll+5ex8tBKVh5cyf6z+5nYayJvDDHd9UUHR5NjzSEqOIrWjVrTNKQp4YHhhAeGExoQSouRLQh6KojEQ4mMaDOCjpEdybBkcDrrNEfSjnD03FGS0pJoFpLf6O5g6kF6zeyFn7cf3Zt2p3+r/gyMG8hlsZcR4BNQ49/Z2xsmTTI9ET7/PAwfLoXromRydxH13ksvmdd774WAyvw+SwmEU1RUFEeOHCEmJoZAV2UTCmw2G8ePHyc0NNTVSRGiSrQG+3AsPPIINGxY88d8bd1rzPp1FttPbC+0PMQvBC+V/zvfOboz5548RwO/BuffYQoM7zic4R2HF/tIa01WXpbz/YmME7SPaM+uU7tYd3gd6w6v44XVLxDgE0B8q3jeue4dWjVqVaXvV5YxY+C552D7dlMKMbx4soWQAELUb9u3wzffmMCh0t0CSgDh1NB+dz969CgWi4Xs7GwCKhWV1V21dU6Cg4OJiIio8eMIUZOWL4eVKyEszDSerm5aa9YdXkf7iPaEBYYB8GfKn2w/sZ0Gfg24qvVV9G/Vn34t+9Elugs+XvmPTT5ePmUHD2VQShHkm19i2zumNzsf2ElKdgrrDq/jp/0/sezAMrYc20LCnwlEBkU61/1h7w90iOxAi9AWVUpDUf7+MGWK6c586lTTM6G3h/QkJWqPBBCiXvvPf8zrXXeZxnmVIlWYCmnYsKEzkEhISKBbt24uTpF7kXMiRPlobR5kwTTurc4CtR0ndzBnyxw+//1zDqUeYsb1M5jQYwIA9/a4lxvb3cjlLS6vUjuEw68fJn1LOlwKxFds20YBjbjmgmu45oJrAFMyseXYFoL9ggHIteYy4ssRpGSncFXrqxjbdSxDOwx1fl5V48bBiy/Czp3wyScwenS17FbUIVXKMlVKXamU2qyU2qqU2qSUurSU9Xoopc4opdYVmB6ryrGFqKpDh2DevPw6n5UmJRBCCFHtvvwS1q0zmTsTJ1Z9fxm5GXzw6wf0nd2Xjm935D+//IdDqYeICYkp1OlAh8gOXNX6qio3Yj677CzHPjgGJ6qacogKjmJQm/zBL85mnWVA6wEE+ASw/MByRi8aTZNXmnD34rtZdXBVlTtR8PMzpQ9ggrisrPOuLuqhSj/xKKUaAQuBB7TWXYBJwGKlVEmtJyOAL7XWlxaYXqnssYWoDtOnm5FNhw+H1q2rsCPpxlWIQsqTuaSUilZKzVBK7VRKbVBKrVJKdXZFeoX7yc2FJ54w8888AyEhVd/nzZ/fzF1f38UvSb/QwK8BE7pPYNW4VRx65BD39ry36gcoIuahGNq93w4uqPZdE90gmi+Gf0HyY8nMuH4GfZr3IT03ndlbZtNvTj8SkxOrfIxRo6BLFzN4n6OnQiEcqlKFaTCwW2u9FkBrnaCUSgYGAN8UWTcCGKSUWm9/vwx4UWt9ruhOlVITgAkA0dHRJCQkVCGJ1SM9Pd0t0uFOPP2cpKb6MGNGH8Cbq67aSEJCRqX3Fb5lC50Bi83m0eekJnj6dVIT6vo5KZC5dL3Weq1SKh6TudRaa51ZYNXuwA9a63vt2z0GvAJ4wBjDoqa98w7s2wft2sHdd1d8e601P+77kbbhbYkLiwPg9s63k2HJYHy38QzrOKzK7RfK0nhgYwB2J+yusWM0CmjEhB4TmNBjArtP7WbOljlsPbGVHk17ONd5e+PbDIwbyIXhF1Zo397epprv4MHwf/9nqvpGRpa9nagfygwglFJ+wMoSPloC7CuybB8QV8K6C4F5WmutlGoMvAnMBW4uuqLW+j3gPYCePXvq+Pj4spJY4xISEnCHdLgTTz8nzzxjRp++5hoYP75X1XZ29iwAPr6+Hn1OaoKnXyc1oR6ck3JlLmmtlxTZLhlplyeA48fze1566SXw9S3/tjl5OXy6/VOmr53OthPbuLfHvbx7/bsAjOk6hrEXj63+BLuJdhHteGHgC4WW7Tq1iwe+fwCAQW0GMbHXRK678LpCvUmdz6BB5j65dCk8+SS8/361J1t4qDJ/rLXWuZgmQIUopf4BWIsszqOEalFa66wC82fsOU1JSqnAgp8JURtSUuC118z83/9eDTuUNhBCFBRH+TOXAFOdCXgWKDGv2R1LpqHulyZVRnWck2nT2pGW1pRLLjlNSMg2yrO7NEsaXx/9mq+OfsWZ3DMAhPuF453i7bq/0SrgFGR0y3BZGpKzkhnSZAjLTizjx30/8uO+H4kNjGVY82EMbjIYP6+y23mMHBnIsmW9mDXLix49EunQoVjlkQqT/53iPO2cVCW35zAwsMiyFsAX5djWG8gGSh6+UYga9MorJoiIj4crr6yGHUoAIURBinJmLgEopcKB74GpWuufS1rHHUumoV6UJlVYVc/JmjUmt9vPDz76KJy2bcve1ydbP2HCtxPItJgacp2jOvNYn8cY2XmkS0ZzdtjyzBZSElIInh5M/Nh4l6VjJCM5k3WG2b/O5o0Nb3Ao9RDT/5jOguML2PvQ3nKdox07YNo0mDWrB+vWQVXHqJT/neI87ZxU5YlnMdDF0ehNKdUbaA/8TykVrpRao5Rqa/9shL1eLEopH+AF4COtta1qyReiYk6ezC99eP75aup5VbpxFaKgw5jMpIJa2JcXopRqCvwEvKK1/rgW0ibcWHY2jB9v5h9/HNq2LX3d1OxU53zn6M5kWjIZ3GYwP476kd/u+40xF49xafAAoK32npDc4NbQOLAxky6bxL6H9jFv6Dy6NenGoDaDnOfIarNy4OyBUrefMgViYyExEf7739pKtXBnlQ4gtNapwDBgtlJqA/AaMERrnQIEAS0BR6/NgcBPSqmNwC/AMeDRqiRciMqYNg3S02HIELjssmraqZRACFFQaZlLy4tkLLXEBA/Paa3nuSy1wm08/zzs2mUaTj/1VPHP82x5LPh9AX1m9WHIvCHO5V2iu7D/of0sHbWUq9tcjXKTzBxnAOFGg7D5ePkwsvNIEick8vq1+V0rLdy5kAveuIDhC4az4ciGYts1aADvmqYk/POfpoG7qN+qVAiltV4BFGuBqrVOApoXeP8B8EFVjiVEVR09Cm+9Zeafe64adyzduArhpLVOVUo5Mpc0pvrSEIpnLL0CRAOTlVKT7ctytNbVUbFQeJhNm8zAZWAa6hYcrP1czjlm/zqb/67/LwdSTC55WEAYR88dpVlIMwBah1WlL+6a4Qwg3PDWUHQE7L1n9uKtvFmwYwELdiygX8t+TOozqVCD6yFD4PbbzfhJ48bBihUyQnV9Jj1eiHrj+edNEfktt0D37tW4Y3sVJu0muV5CuFppmUsUzli6tfZSJNxZZqYZcyAvD/72N7j8crP8bNZZXlz9IjMSZ5CaY6ostQlrwyOXPsLYi8dW26jLNcZRSdsNA4iinrziSUZ3Hc3r61/n3cR3WXlwJSsPrqRdeDue7f8swzsOB0z1peXLYdUq08WrY6wOUf94wGUtRNXt3w8zZ5pmCs8+W807lxIIIYSotMceg927oWPH/FIIAC/lxTub3iE1J5XLW1zOV7d9xe6Ju3mg9wPuHzzg3iUQJYlpGMO0q6eR9EgSrwx6hdiGsew+vZvkc8nOdSIi4AN7fZJ//cuUHIn6yUMuayGq5rHHwGKBO++Eiy6q5p1LGwghhKiUTz81dev9/DT3Pb+aUV/fSpbF9O4eGhDK29e9zbq717Fq3Cpuan8T3l6eU2fG0wIIh4b+DXm0z6POBtd3dbvL+dkLq15gcd79jBp/FosFhg93DoUk6hmpwiTqvGXLYNEiCA6GF14oe/0Kc1RhkgBCCCHKbedOuOceDSga3fQMD255BoDBbQZzT497ABjVZZQLU1hFjs6MPfTW4Ovty8jOI53vc625TF83nVOZpyB6DqGtt3PgQBvGjtV89ZWSQvh6Rv7cok6zWEydWjDd0DVrVgMHcVRhkjYQQghRLtv+PMJlA0+RkaGg0zxOdHiG5g2bM23gNIZ1HObq5FULbfPMEojS+Hn7sXLsSsZ3G49fgI3UG66GgLN8/bVi6D27sdqKDv8i6rI6clkLUbJ33jED4LRpA488UkMHkSpMQghRbhYL9L32MClHI6DJr/S+/30+H/Y5+x/az+N9H6dRQCNXJ7FaeGoVpvPpENmBmTfO5NDDh/jnTaMIuWM8eFlYPLsdT0wrfRwJUffUoctaiMJOnoSnnzbzr74K/v41dCBpRC2EEKXKzsvm460fs//sfmw2uPtuOLfrEgIapbBoEaz/63KGdxyOr7evq5NavTy8CtP5RDeI5tn+z5L89lzueHI1AK/+sw1Ll5rPp6+dzsYjG12YQlHTpA2EqLOefBJSUmDwYLj++ho8kHTjKoQQxew+tZsZiTP48LcPOZN1hom9HiRgxet89JFpk7bix0b06tXN1cmsMXWtClNJgv2C+fjf/WkFPP+8Ytgw+PjrJCatnIRG0zumNw/0eoDhHYcT4BNQ1u6EB6nDl7Woz378EWbNAj8/eO21Gm6eICUQQggBQK4tl8+2f0b/D/vT/q32vLruVc5knaFbk+4c+uoeXn4ZfHzgyy+hV0kjhdQhcS/G0e79dlA3amSd13PPmUHm0tNh7NAYbm/8KmEBYWw4soExi8bQ7JVmPPj9g2w5tsXVSRXVRJ54RJ2Tlgb3mA48eOYZaN++hg8obSCEEAKAd/e9y8gvR5LwZwJBvkGM7zaeDeM3MvjPRL5+vzPe3mYk48GDXZ3Smhc1LIqmdzcF9x+yosqUMuND3HQTpKR48c2Uv/FFn6PMunEW3Zt252z2Wd7c+CaXvn8pKdkprk6uqAZShUnUOY8/DocOQc+eMGlSLRxQunEVQtRDyeeSmbdtHu0j2nPdhdcBcHX01ezN28t9Pe/jjs53EOgVyj33wNy54O0Nn3wCw+pGJ0uiCD8/mD/fjCo+fz7cOCSAb7+9i8QJd7Hl2BZmbZ5Fni3P2UjeYrXw4JIHubn9zQyIG4CPlzySehL5a4k6ZdkymDHD/JB98IEpKq9x0o2rEKKeSM9N55vd3zB361x+3PcjNm1jQOsBzgCifUh7frvvN5RSpKXBdbeY3+WgIFiwAIYMcfEXqEXJs5OxZdsgztUpqT2+viZI9PeHjz6Ca64x9+QxYy7mjSFvFFp3yd4lzEicwYzEGUQGRTLsomGM7DySy2Ivw0tJhpy7kwBC1BmpqTB+vJl/+mno1KmWDixVmIQQddzyA8t5c8ObLNm7hOy8bAB8vXz5S7u/MKbrGOd6SimUUuzdC7feCr/9BlFR8N13plS4Ptn/xH4sJy2w0NUpqV0+PjBnDoSGwptvwtixsHkzvPyyCTAcujftztQrp/Lp9k/ZfXo3b296m7c3vU1sw1hGdBrBc/2fw9+nprpPFFUlTzyiTtAaxo2DgwehRw9TjanW2KswSSNqIURdcTbrLMfTjzvf7zm9h692fUV2XjZ9mvfhrSFvkfxYMgtvW8hf2v+l0Laffgrdu5vgoW1bWLu2/gUPAE3GNaHZ/c2gHj4De3nBG2/Ae++ZoOH11027l+P5lxTNGzbn6fin2fnATjZP2MzkyyYT2zCWpLQkFu9ejJ+3n3Pd9YfXk2fLc8E3EaWR3vRQeAAAGuZJREFUEghRJ7z6Knz1lcnx+PzzWqq65OAogZAqTEIID6W1ZsfJHXz/x/cs2buEVYdWMbHXRF695lUAhnYYisVqYWiHocQ0jClxH5mZ8PLLF/Ldd+b9rbfCzJnQqB70QlSSNtPaAHA04aiLU+I699wDF10Et9wCK1ZAx47w9tumVMpBKUW3pt3o1rQbLw58kV+SfiE1OxVlv6f+mfInl866lLCAMK678Dr+0u4vDG4zmBD/EBd9KwESQIg6YPXq/BKHOXPMqNO1yhFAeHvX8oGFEKJqVhxYwee/f873f3xPUlqSc7mX8uJM9hnn+6jgKB685MFS9/Ptt/Dgg/Dnn83w9zfdZ997rzQNE9C3LyQmmqpMy5bBbbfBlVdexIIFEBlZeF0v5cXlLS4vtOxI2hHahbdj9+ndfLz1Yz7e+jF+3n70je3L1XFX80DvB2jo37D2vpAApAqT8HAnTpgfI6sVJk82XcjVOmlELYTwAJmWTH7a/xPH0o85ly3evZgZiTNISksiKjiKMV3H8Nktn3Fq8ik+vOnDMvd56BDcfDPccAP8+Se0aZPOhg1w333yk5i2Po209Wn5I1LXYzExZnymd981gwj+/HMU7dqZQDM39/zb9m3Rl10Td7HrgV28NPAl+sb2xWK1sOLPFTzz8zP4euU3rPhh7w/sP7sfrXUNfyMhJRDCY2VlwdChcPQoXHEFPP+8ixIi3bgKIdxQanYqG45sYOXBlSQcTGD94fVYbBZmXD+DCT0mADCy00jCA8MZ0nYI3Zp2K3fvN8eOwbRp5oEwOxsaNDCDiXXunEiXLlfW5NfyGL9e+Ss6R8NSV6fEPShlSqUGDYJhw86QmNiYRx4xDa2nTTP38/MFne0i2jE5YjKT+07mdOZplh9YTlJaEoG+gQDk2fIY/sVw0nLSaBbSjMtbXE7f2L70je1L1yZdpZvYaiZnU3gkqxXuuAPWrIHmzeGzzwr37lCrpARCCOFiNm0r9PDfZ1Yf1h9ejyY/J9ZLedGjaQ+CffNHNruk+SVc0vySch/nyBGTa/zWWyYTB2D4cJg+3eQyJyRIzq+To+RB8pYKad0a/vOfrWRkxDN5MuzaZdrLdOpkahKMHFn2/Tw8KJxhHQsPKHI26yxXx13Nij9XcPTcUeb/Pp/5v88HINg3mE+GfuJs8J9rzcXXy9fZzkJUnAQQwuNoDQ89ZBpNN2oES5dCs2YuTJB04yqEqEUZuRlsO7GN3479xtbjW9lyfAs7T+7k2KRjzp5rgnyD8PX2pWt0V65ocQXxreK5ouUVzkG8KkJrWLnSBA0LF+Z3PPeXv8DUqXDxxdX45eoQbbMHU3JrKEYpuP560zPTzJmmBsH27TBmDEyZAg88AKNHQ9Om5d9nZHAkXwz/Apu2sfvUbtYkrWH1odWsSVrD3jN7aR3W2rnuE8ueYN62eXRv2p0eTXvQo1kPujftTmzDWAkqykkCCOFxpk0zvTj4+8PXX5teHVxKunEVQtSA9Nx0sixZRAablqa/JP3CmEVj2HdmX6GSBYcdJ3dwcRPzND/3prlEBEVUqR/9P/4wpbvz5plcYjCjSd96KzzxhOkyW5RMaw32wmkJIErn6wt//asZw2nePHjpJdi501xfTz1lBqIbO9YMQBgUVL59eikvOkR2oENkB8Z3N4NDHU8/TkRQhHOdXad2cTzjOEv2LmHJ3iXO5SF+IYzsNJIZN8wAwGqzkpyeTExIjAQWRUgAITzKf/4DTz5pci8+/ti0fXA56cZVCFEFB84eYOepnew5vYfdp3az+/Ru9pzew5FzRxjfbTwzb5wJQHhgOHvP7MXHy4eLIi+iS3QXukZ3pWt0V7o37U54ULhzn6V1tXo+Nhts2gRLlsA335iecxyio2HCBDM1b17lr1z3OYIHZZ/Eefn5mUBh9GhTq2DWLJNB+N13ZgoMNG0nbr4Zrr22cDew5RHdILrQ++9u/44DKQdIPJrI5uTNJCYnsuXYFk5mniwUnP9x5g86vNWBEL8Q2jRuQ5uwNlzQ+ALahLWhTeM2XBJzCcF+wUUPVy9IACE8gtbwzDNmUso03Lv1Vlenyk6qMAkhSmDTNk5lniL5XDIHUw9yMOWgeU09yPs3vE9oQCgA9393Pz/s+6HY9n7efuRYc5zvL2h8AVvu3UKHyA6FBtmqLKsVtm0zXWGvXg0//QSnTuV/HhJiHthGjICBA13YzswDSfWlyvHyMqUNQ4bAyZPwySdmYMING2DxYjOBaS9x1VUQHw+9e5tqzBXJw1NKERcWR1xYXKG2FCczTpJrze8W6nj6ccIDwzmddZotx7aw5diWQvvZfv92OkaZahDT105n58mdxIbG0rxhc2Ib2l9DY2ng16DS58RdSQAh3J7WZpyHl182Py5z5sCdd7o6VQVIFSYh6g2L1cKZrDOcyjzFbym/cXbnWY5nHCf5XDIXN7mYmzvcDMDqQ6vp/2H/UkfPnXLFFLo26QpA75jeWGwWLmx8Ie0i2tEuvB0Xhl9Iy0YtC/Uc4+3l7dymojIyTB3z334rPKWnF16vVSuTw3vttSZoCAys1OHqr5degl690H36AaC8VfHqZitWwMaN+QMYiRJFRsLDD5vpyJH8AGLVKnMtb99uRrgGaNLEjHbes6cZBb1DB3MtV3RQWUd1QYcrW13JycknOZ11mn1n9rHv7D72ntnrfI0Li3Ou+/0f3/PTgZ9K3O/QDkP5cviXgGnsPTVhKlHBUUQFRxHdIJqo4CiOZh0lPTedYN9gj6guJQGEcGvZ2aY/8Q8/ND8E8+bBsGFlb1erpAqTEB4jz5bHuZxz+Hr7OnMFD6Ue4pekX0jLSXNOqdmpnM46zems03x3+3fOHo76zOpDYnKBuj2/5c+O7jraGUCEB4aTZ8sjLCCMpiFNaRHagpahLc3UqCXNG+bXA3q2/7NV/l5ZWaZr1eRkSEqCffvyp/374fBhkxlTVFycGejr8suhXz9o1046lKuSXr1Mt1QfzgdU8QBixQrz+fz5LkuiJ4qJMW0l/vpXyMmB9eth+XJTcpaYaK79b781k4OvL1xwAVx4obmuW7aE2FhTBS82FsLDy3etK6WICIogIijivD2WTek3hVs63EJSWhKH0w5zOO2wcz48ML964eG0w7y+4fWSd7IBVoxZQXyreADe2vAWP+z7gbDAMMICwmgU0IiwgDDCAsNo3rA5V7W+yrnp8fTjBPsFE+QbVO7umKtCAgjhto4cMf1Cb9hgcsHmzze9NrgdRzeuMhK1EAAopa4EXsXcY3KBiVrrdUXWUcCzwHBMh5ebgXu11hnn23emJZP3Et8j05LpnLIsWWRaMmno35BpV09zrjtg7gBOZpwk05JJhiWDtJw0Mi2ZALx89cs8dtljAKw5tIbbF95e6jFTslNoHNgYMDmUEUERhAeG42PxIa5JHNHB0TQNaUrvmN7ObS4Mv5Csp7II8Ako51kzhZlZWWZKTYWUlMJTwWWnT5tgwRE0pKaef98+PiZXtmvXwlNF65KLMvTvD/Pno28dDXwE2krLuR+Zop6AANNH6fz5Zj1RKf7+JtjtZwp50NoEyZs2menXX2HPHhNI79xpppIEBJi2PRERprQjIiJ/ioyE0FBTja+kKSCgePAR3yre+eBfkNa6UFXEyOBIpg+azomME5zIOMHxjOOcyDjBodOHSLWmEhWc/0+5KXkT3+z5psT092jag00TNgGmsXeTV5oAoFAE+wXTwK+Bc5p65VRnF7YrDqzgs+2fEeQbRKBvIIE+gc7XIN9ytlRHAgjhptatM8FDcrLJNVi0yI27CpQSCCGclFKNgIXA9VrrtUqpeGCxUqq11jqzwKpjgCHAxVrrLKXUB8A0YOL59n/yeDbvTv4VZ+tU7Xj1JcTfj0Wrz6K1eajI+eUyfHKzOBl6hjS/XEDRND2UljkR7DzRiVk7zXpJh3px2+a5+HkF4Ofth6+XH37e/vh7BxLgHcCCR614e53FZoMbrZ9wnVWR2TCA7aePExPTCq+0XPyOZbDW15dvG4HFAjoHwpOzyLNmYbVCXh7kWSAn1+Sg5uZAtv3Vsex0ni/7MaUivljpRBoWvNhOqPP7dyYFnwI52oFAHODjDY3DoXFj8wDUrBk0awrNWnkRd2MosbEmRzZlVQraogm9PBQvP5NLmf5bOpbTlgr9nUvaPrhLfmPSrP1ZZP+ZXaF9BncJxi/Cr9D2Aa0DCGxt6lHlnswlY9t548tiStreN8KXBl3MebZmW0n7Ja1C+yxpey9/L0L72v9O/fuTOmkO/ANUdiat5swxFfmtVvj+ewkeqplS0KaNmW67LX95RobpSWzPnvyAwjEdPmwC8YMHzVRR3t5m8MSAABPQ+PsXni/8XuHvH4C/vwnkvb2b4O39CN7eEOENTXzM/pKSDhAX14pFM+Fb+7KozP/jnryHybZmkGXNINuWQWZeOpmWDCIzI/jwQ/P9c615NNx1P9nWbHKt2aQrTToalAY0P51rjGW7Wfe7P9L5YMsJKPC549XHu/xhgQQQwq1YLPDCC2ZE07w800Bq/nyTG+C2pA2EEAUNBnZrrdcCaK0TlFLJwACgYFbabcAMrbV9ODL+C/xEGQGEPh7M9OO3lb7Cj/l1iv7NAACeoDPrMVUIruEAoznI7JWtGG9fry+h/JvYEnaWY5+KZ+/PphUf0cq+fSr/5ndWE84MOgMQiYX5Bes3lcNqwvk/2hBIFq04zSscJRXNh5ylESk0IoVr6IQfJTSgtgIn7FMB/pwg7pH88/U7C7EQxmXcjB8pAOznBc5waYXSWtL2nXmCeNYDcIxxHGR0hfbZmScIL7J9K/uZBkilL7/z7wrts6Ttw1lNZ/4JgIVIfqNi1YlK2t6fE/Qh/zzvYiEQhiIPpbWpjwtw9dUVOlZdFV8LxwgGLrZPJTlHA04SySkinK+O6SSRpBLKOUJKnHKt/qSmll3yVzGtS1jW1D6VLP8H1R94u9T13vgC3nC+u8E+FWdabJUvM1QCCOE2duwwg8hsMiVyPPKIGfPB7Xv+kBIIIQqKA/YVWbbPvvx86+0DGiulQrXWhW7LSqkJwASAaNpyECuqQC58WfOXsJw25OKFjdYEkUwwF7CRcSxHoYnCn2M0LnU/Bd8rNArNpSzlAo7ii4VGhHCOtlzITt7hLXyx4IcPOfRzru+YvLHihc35auZteGHldrYyhQ8AyKURO/gnkaSykPw2Ett5hjzK36OLb5HgJ5Rt5NEARX7j7mD2YyspKDmPkrb3JT8nP4BjNGJzhfZZ0vYBHCvweWqF91nS9g3Y71zmhaXC+yxp+9LOcyQrK7RvUXtCSCeEdOI4UOFtc/ElnQbk4F9oyiag2LKCy614O6c8fAq9L+9yG17OXxSgyC9M1aclZXx3BwkghMtlZJjxHV580RTjt2hhelrymFJe6cZViIIUJj+8oDyKd2hZdD3HE2mxfySt9XvAewA9e/bUYzYNqFCC7qjQ2uWXkJBAfHx8Ne91NgB+FMw5fcY516lS+yxp+0edc20qtc+Stn/UeU7On29a9j7zt89f1ojSc5PLs8/C25d0niuijL/Tt9+aHj+yC1TjCgiABQvctDFf7aqZ/53a4wc0ruZ9uss5KW9eqDzxCJex2Uwfz+3amfEdcnLgrrtMv+QeEzyAVGESorDDQIsiy1rYl59vvRZAOtjrxQjhyQICzL0hIMCUThd4L0RdIE88otbZbKZRdO/eMGqU6W2pe3dYudKMPtmwoatTWEFShUmIghYDXZRSnQGUUr2B9sBypdQapVRb+3ofAeOVUo66Mw8CC7UuqbNRITzIihWmt6Xvv4cFC/hz3DhT8vD992b5ihWuTqEQVSZVmEStyc01DaJfeMG0dwAz+MsLL5jh6z02A1+6cRXCSWudqpQaBsxWSmlM1aQhQBDQEpxdCs0FLgA2KKXygB2U0YBaCLdXcJwHe1H6wQYNaO2omjJ/frHPhfBEEkCIGrdnD8ycaQaDO3nSLIuNhcmT4e67Iaj83Q67J3sVJimBEMLQWq8AepXwUfMC61iBp+yTEHXDxo3nDw7s40SwcaMEEMKjSQAhakRSEnz5pSm1/eWX/OWdO5vele64A/wq1umH+3KUQHhsEYoQQohq8fjjZa/Tv78ED8LjSQAhqkVenhkx+scfYckSM+8QFAQjRsCECabdQ53LqJc2EEIIIYSoRySAEJWSmurD0qWwfr2Z1qyBtAKDeQYGwnXXwa23wpAhZuj3Oku6cRVCCCFEPSIBhDivnBzThmHnzvwpMRH27r282LoXXgiDBpmBNgcMgOBgFyTYFaQbVyGEEELUIxJA1GN5eXDqFJw4AcePm3YLBw+a6dAh8/rnn/lV/Avy87PSq5c3l1xiqiX16WMGgKuXpAqTEEIIIeqRKgUQSilf4G/AC8CdWuvPSllPAc8CwzEjj24G7tVaZ1Tl+PVRXp4Z2DInx7wWnM/IMNWISptSU/MDhpMn4cyZso/n5QVt20KHDvlT585w5sxqBg68sua/sCeQblyFEEIIUY9UtQTiHkAD68pYbwymH/CLtdZZSqkPgGmU0ef30T3nmHrVShzDCmlUOeZBa1Vo3jEqUWXnU9M0n4cklrjv0uatNoVVe5FnU1htXli1Is/mdZ55+/rW/GUWqzfZed5k5/mQY/UhO88Hq636qsl4KRvhQVlEBWcQFZxJ84ZptGyUSstGqbRolEaL0FRah6Xg72PN3yjJTAcOHIB1q6otLR5t715ASiCEEEIIUT9UKYDQWr8NoJS6oYxVbwNmaK2z7O//C/xEGQFE8rkQnlnRrypJrHO8sBJAdqHJnxwCyCaITBqSdt4pglNEcpIoTtBYn8E7wwaVKAdqXf1fzeNZ602jDyGEEELUZ2UGEEopP2BlCR/101rnlvM4ccC+Au/3AY2VUqFa69Qix5sATABo7NuSUe2/QqGdXX8qU95gX5cC8zp/H2jne+V8X3x7imyvStneas3Dx8f7/McvcgwvZcPHy4aXsuGtbHgrq/3VhreXeTXrWPFW9vWVtcD6Nvy88/D3tuDnZcHf20w+XiU0SCi3QCAWiCUTyKzCnnJzc/GrMwM5VF1OZCRHmzcnLSHB1UlxK+np6STIOSlEzokQQghPV2YAYQ8SLq3icRSm7YNDnv21WH0crfV7wHsAPXv21P/ddHMVD111CQkJxDuGoReAnJOSHJVzUoxcJ8XJORFCCOHpaqvfycNAwT56WgDpQEotHV8IIYQQQghRDWokgFBKhSul1iil2toXfQSMt1eHAngQWKi11iXvQQghhBBCCOGOamociCCgJRBqfz8XuADYoJTKA3ZQRgNqIYQQQgghhPuplgBCax1f5H0S0LzAeyvwlH0SQgghhBBCeKjaagMhhBBCCCGEqAMkgBBCCCGEEEKUmwQQQgghhBBCiHKTAEIIIYQQQghRbhJACCGEEEIIIcpNAgghhBBCCCFEuUkAIYQQQgghhCg3CSCEEEIIIYQQ5SYBhBBCCCGEEKLcJIAQQgghhBBClJsEEEIIIaqVUupKpdRmpdRWpdQmpdSlpawXrZSaoZTaqZTaoJRapZTqXNvpFUIIUTESQAghhKg2SqlGwELgAa11F2ASsFgpFVTC6t2BH7TWHbTWvYFFwCu1l1ohhBCV4ePqBAghhKhTBgO7tdZrAbTWCUqpZGAA8E3BFbXWS4psm4zcl4QQwu259Q91YmLiKaXUQVenA4gATrk6EW5Gzklxck6Kk3NSnLuck5ZV2Vgp5QesLOGjJcC+Isv2AXFl7C8aeBa4u4TPJgAT7G/TlVK7K5zgmuEuf0t3IuekODknxck5Kc5dzkm57g1uHUBorSNdnQYApdQmrXVPV6fDncg5KU7OSXFyToqrK+dEa50LFGvboJT6B2AtsjiP81SZVUqFA98DU7XWP5dwrPeA96qU4BpQV/6W1UnOSXFyToqTc1Kcp50TaQMhhBCiOh0GWhRZ1sK+vBilVFPgJ+AVrfXHNZw2IYQQ1UACCCGEENVpMdDF0ZuSUqo30B74n1IqXCm1RinV1v5ZS0zw8JzWep7LUiyEEKJC3LoKkxtxu6JzNyDnpDg5J8XJOSmuTp8TrXWqUmoYMFsppTHVl4ZorVOUUrGY+rWh9tVfAaKByUqpyfZlOVrrK2s94ZVTp/+WlSTnpDg5J8XJOSnOo86J0lq7Og1CCCGEEEIIDyFVmIQQQgghhBDlJgGEEEIIIYQQotwkgBBCCCGEEEKUmwQQFaSU6qiUOqOUmurqtLiaUupepdRvSqlNSqmtSqm/ujpNrqCUulIptdl+DjYppYr1jV/fyLVROvkNqZvk75pP/v8NuTcUJ9dGyTzx90N6YaoApVQj4C3gU1enxdWUUt5AW6Cv1jpdKRUD7FVKLdZaH3Fx8mqN/ZpYCFyvtV6rlIoHFiulWmutM12bOteQa6N08htSN8nfNZ/8/xtybyhOro2Seervh5RAlJNSygv4EPgHcNLFyXE5rbVVaz1Ja51uX3QayAW8XZgsVxgM7NZarwXQWicAycAAVybKleTaKJn8htRN8nctTP7/neTeUIRcG8V58u+HlEAUoJTyA1aW8FE/YCrwP631L0qpQbWaMBc63znRWucWeP8a8LnW+lDtpMxtxAH7iizbZ18ujPp6bRT1b+rhb0hdIPeG4uTeUCa5N5Stvl4bBXnsfUECiALsP3rF6igqpW4BWmit/1H7qXKt0s5JQUqpfwMxwC21kij3ogBrkWV5SOkeUO+vDaf6/BtSF8i9oTi5N5RJ7g3nUc+vDcDzfz8kgCifa4EOSql19vfNwTR60VoPc12yXE8p9TLQBrilSK5TfXEYGFhkWQvgCxekxa3ItVGI/IbUTfJ3LYX8/8u9oTRybTh59O+HjERdCY5W8lrrqa5NievY6+29DYQBd2it81ycJJdQSoViiqX7a623KaV6Az8ArbXWKa5NnWvItVE2+Q2pm+TvKv//DnJvKE6ujfPztN8PKYEQlTUEuBfYBKxWSjmWT9FaL3NZqmqZ1jpVKTUMmK2U0pgi6iH19QZhJ9eGEPWX/P8j94ZSyLVRh0gJhBBCCCGEEKLcpDGPEEIIIYQQotwkgBBCCCGEEEKUmwQQQgghhBBCiHKTAEIIIYQQQghRbhJACCGEEEIIIcpNAgghhBBCCCFEuUkAIYQQQgghhCg3CSCEEEIIIYQQ5fb/9OoP4zkR5r0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"스텝\")\n",
    "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"로지스틱\")\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"활성화 함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"도함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAEKCAYAAABTxSI1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuUXHWd7/33N5fu0AlJOhcSQ+gEItLjKPBwO4zjDByV8QyznAOIy9uJg3LMzBEFRpEHZ0ZlZI6XKAreRpklqEGO5wwEUI+3KKIOyMMlKg9HwzxGEggYbjYJSSd9/T5/7CqoVKq7qvb98nmt1SupXbv2/qa78utP/fb+7m3ujoiIiIhka0bWBYiIiIiIQpmIiIhILiiUiYiIiOSAQpmIiIhIDiiUiYiIiOSAQlnOmNnnzWxbw9ffNTz3LTO7vMvtdfUaM7vdzP5mmuffambPNHzd3fDc6Wa2p832l5mZm9kfdVpTh3V/2cw+21yDmQ2a2cua1n25ma1pePyAmZ3Xxb7czE7qsr5tZnZuN68RKRqNX9GY2Xlm9kAS245C41d6FMpywsxOM7NtwJlNT62r/YdY2eb1XzSz6zvc129qA0v9a8pBrJm7X+fuCxu+Tun0tTXnAJPAr7t8HQBmdllj7R285A3Ah5qW/RPwmg729Ukz+0aHdZ1RC3f7zexnZnZ8B6/Z1vRzaP6a9mcukhcav7pjZn9tZr/qcN1PtBknvtriNfvbvOaFTetr/MqJWVkXIAF3/zGw2syWA6cAhwK/B/7N3Z8FMLPpNrEceKC23ixgXm35cz9jM3sx8GLgAxwYyMdqn4J+ONXGzWwAuH+a/V8O/GK6As3sWODDwFPAF8zsr9x9ZLrXNL1+PnAd8PWGZQuB3Z1uo0uTwHAHdQ0CtwJ/B3wDuAD4npkNuvtQm5efB9w0xXN7Oy9VJDsav0LZ1+F6HwA+OsVzVzH1OPEa4N+ali0CtjYu0PiVLwplOWJmrwa+CTwIPA0MAAvN7D+4+/9XW21OLYjsdfex2usOBU4H5tUGtP9U207dXbU/DwNeMk0Jd071hLs/XKtlFvAq4CiCwek2d3+qVsfptT8XBi/xXbXHM4C/AT4OfKb25zeAe83s74Fveu0qxmY2F/gVcJO7v7upjPuBVS3KO3Kaf1MUhwPbO1jvQuBH7n5V7fF7zOwvgbXAp9u8dsTdpz1kIlIEGr/ajl+NVgAPT/N8Y+3DTPHh0Mz6gMemeOked3+maf1Wv/M1fuWIDl/my5HAZuBkdz+d4BPnJMGnyLr/GxgCXtuw7NPA7QSfvD7l7t9yd3N3A/53fSV3v53gk14f8FcEn4heBzzh7pe7+1T/uQEws0XAL4G/rm3j5cCvzexVDavNrdX3eO017yIYfN4LrHX3y9z9aeA04KvAtcAjZvZntdePEQzqjzTv391X1/5NPcDS+r/R3bdNVzfwyqZDnqe1Wb8+EJ9GZ/9HTiH4/jf6UW25SFVo/Jpm/GpyWm3dqPqBJyJuQ+NXjmimLF++SDAYfN7Mzgb+mWDwusPMDqmt84/ufjk89ynpU8CfAC8jGNS+b2bfBC52960c7O+AVwMvd/dHzewE4DYze8jdv1NbZ5aZzQHG3H2i4bXnApPufnZ9gQUnxr4b+EFt0V53n9fwmnuAi4FvuPtofaG7jwMfN7PPEHxy/Ult+SjwZ0zBzN5LcI7YTDO7n2BQniA43DHVJ7ZNwF82PP72VNtv8MfAfOC/mNk/1T81N/67aodj/lttvaebnn8K+MMO9iNSFhq/2oxftX0eRhAIx83sCHdvF+Cms4hagIxA41eOaKYsJ8zsa8D/C1wB/EdgIcFJ6v8C/JzgE2azbwBLgFPc/YnauRsvJziP4PQpdvVigqnqRwHcfTPBuRyNhwU+QzBAvqvptQ8CAxac1NtrZkfVat0y1b/L3e9y9xsbB7Sm5/fXPhnvn2obdbUB+APASQSfdO8DPktwuOC107x0sraf/bX9TLbbF8H3+1+AfyU4b6PZfyT4lHot8CiwuOn5JbXl7fyPaU6UfXkHrxfJnMav9uNXgwsJvlf/TDB+RbEC2DHFcz9qHlOAJ1usp/ErRzRTlh/vAGYTBIZxYLj2aew5ZvbHBFPrdWfXT6Ktq30y/FjDop8Av214/HXgOgs6pX5DMAgeB7ytYZ3/5u5faC7Q3X9sZu+obX9VrZZvAP9YW+UZ4KcN9X6Z4DBDp/7c3b87zfMvBn7t7v+ntv3/BWxw93m1fcXCzN5OMHV/HsHP4n4zu8TdP9Gw2nPna1jQwv5KgnNNsGAK7ZXAJ9vs6lSm/z8Y9bCESFo0frUfv7DgUjqXAn9BMAt3v5n9g7v/Uxf7qW9rEUF4anVu2lyg3llxJfBC4D/Xn2z62Wj8yhN311eOvggGjJ3TfF3S4jUzCD593UXwSWiSoPPpu8CrW6z/MoIZHgf+O7Cm4bnbgb9pU+N8YEHtz/kEnVbzal9zm+qa1fR1XG2/i1s8Vz9f7HvA37bY7x8SHKKsH1q8nuCEWoAvE3zqPJ0gMNVfc3ltf81fFzes8wBwXu3vawm6hv6s4fn/C9gFXF2r0YGTGp4fJPhk/m6CE4g/RTAg9Tessw04N+v3l770leSXxq9px6/jCE7K/1jDspMJusfXAzMJPgg+0OH3+mUE56XNbrPeVcC3pnle41eOvjRTlj8fovaJpYWv8nyreKNPEJxbdRHBJ8vdwAuA1wPfMLOz3f2586jc/U4zGwbeShA0emuHBjv178DS2t+t6c+99RrdfZKmQ4VmVj/HY8KbPknXnp8NrCGYlj+Au/8fM7uIoAV7EXAbwSA2nQ8RXJes2UTzAgsuuXEp8GZ3/37Dfn9uZqcR/ILw5tZ+d99iZq8hGPw+QnCy85/5FO3kZjaT579fbbX6PonklMavFuNXrXHoywSnQ1zW8G+5p9Zo8G6CUNb4mpkEs49T+UOCsDSztu44QWBsHlss2NzBnZfuPq7xK18UyvLnSuDNHHziZd33Wyw7E7jK3Tc2LNsOrDezVwJ/DnzbzH4JHFN7vv6zf5jgU+njQOPr2/nP7v6txgW1lvJvtV69M+6+l2CqfarnvwR8ycxm+oEn8U61/kED6zTr7jazY7320bDpuV8wzXWM3P0HTN+u32grrS/t0ZKZHeLdnbMikhWNXy3GL3efNLM/9uDyFs3P3U1w/l3ztdzWElyXsZ369c4+VtvOVGPLQd2e9bFF41d+KJTl09fcveOrVAPfAd5lZr8huFbPHoJr+pxD0Hr9udp6Lyf4NOYEQWVf86eY2iDYibkWXM+nUatPwYnoJJCF3G4ndwmIuo/VSe9DJEMav1poFcjarP9lgtm1blzWfpVoNH4lS6Esn843s/8yxXN3u/srmpZdQvDJ8n0E5wQsIjhp9T7gnPrUvzedVBvR16dYrqs4i1Sbxi+RkCyFiQERERERaSPydcrMbLaZXWJmY2b2hinWWWbBDWd/bWZ3m9lPzeylUfctIhKFxi8RyZM4Lh77doJj/HdNs84JwPfc/Q/c/RTgFoITQkVEsqTxS0RyI/I5Ze7+eYBaS+1U63ynadHvptq3ma0D1gHMmdN74sojDotaYmzcZ2GWn+7eOOsZnZwFE4ZNdNSoOKWZs2YwMR5tG3FSPdPLWz0A2x/Z9pS7L22/ZnRJj19H5Gj8gnTGsPEuPuvP8JlMWiI9O12Z8KDmWW6MWzan9HiL/4azbAbjrZ6IZYcdX9XiObNnGGOT3X1/kv52zpphjCc8hlkXp3lFHb9SP9HfzJYRXMvm/FbPu/s1wDUAR79owG/8QX7OeXtsy9+yYnB91mU8J956xtgw9DI2XXsqK256KPRWzrn4ZDauvyemmqJTPdPLWz0A2/nM9qxrmEo349eLXjTg3/xhfj7EAWzbcgmrBz+ayr5u3N3+0mFrHjmLrUfckkI17W3aOcibho/nhr4pr3yTuG07Dvxd/rfzVnHlnuT+O/Q+3NPV+he+8HA+/ZtO7r50oEO3J/d7/Pw/OZwv/fRRFmwdSWwfdT1bprqj1fO28/lIP7BU731pZosJbgZ9ubv/OM19S3tr++/kjLfdxYOXdnwJGpHK0PjVnXPnb866hK6csXzKW2CmZvXKVremTM7IwCgjAy1v6xmrZ1cZz67qfmauG7vW9Ca6fYDRwZWJ7yO1UGZmLwB+CFzp7tentV/pztr+O7n5rKsUzEQaaPwKp2jBbP7s/ZmHs7SDGZBKMANSCWZJh7Okg1kioczMFpvZHWZ2dO3xKoIB7Qp3vyGJfUq86sFs30sOz7oUkVRp/IpX0YIZZD9rtnrlk5nMmqUh6WAGyc+aJRnMkpop6yO4DcOC2uMrgWXAe83srtqXpv9z7uazruL3F+xVMJOq0fgVMwWzcHp70j0nUcGsc0kFs9hCmbuf7u5fr/39EXdf6e731h6f6+6L3f3Uhq/T4tq3JGfDcdfx+wv28thrj8y6FJHEaPxKnoJZOJoxC6+IwSzVE/2lmDYcdx1nvO0uBTMRieTc+ZsLF84UzJJTlmAWZzhTKJOOqDNTROKiYNY9dWaGV6TOTIUy6Zg6M0UkLkUMZlmHM3VmhleUzkyFMumaOjNFJA79M4ezLqFreQhmOpwZXhqzZlEolEko6swUkTgUbcYMsg9moPPMoshzMFMok9DUmSkicVAwCyftYDbZk859cqsczBTKJBJ1ZopIHNSZGY5mzMLLYzBTKJPI1JkpInFRMOueOjPDy1swUyiTWNQ7M0eW92RdiogUXBGDWdbhTJ2Z4aXRmdkphTKJ1ZqFT6gzU0QiK1owg+xnzdSZGU0egplCmcROnZkiEgcFs3AUzMLLOpgplEki1JkpInFQMAtHwSy8LIOZQpkkRp2ZIhIHdWaGo2AWXlbBTKFMEqXOTBGJi4JZ99SZGV4WwWxW6nssqMnxCXpmXc3k+AQzZs3MupxCWdt/J2vPupOzuZhj1m/PupxKuO3NX2K0r/0tbL47BLy9/fZ6hvt4xdfOj16YZGZifILZMz/NxPgEMws8hp07fzM37j4h6zI6Vg9mm3YOZlbD6pVPsm3H0lT3OTIwSu/D4bvxt05+kAmebbveO7YBR7Tf3qyJQznuscu7rqMezBZsHen6tWFopqxD+4Z2M8N+y76h3VmXUli6Z2Z6OglkWW5P0rdnaA9mv2XP0J6sS4msaDNmkP2sWdE6MzsJZN0Ynxlte2nNmimUdWByfILRPXsxc0b37GVyfCLrkgpLnZki6ZsYn2D42WHMnOFnh5kowRimYBZOkYJZ3qQRzBTKOrBvaDd47YGj2bKI1Jkpkq49Q3sOGMPKMFsGCmZhKZiFl3QwUyhroz5L1kizZdGpM1MkHfVZskZlmS0DdWaGpWAWXpLBTKGsjQNmyeo0WxYLdWaKJO+AWbK6Es2W1SmYda+snZlpSCqYKZRNo9UsWZ1my+JRv2emgplI/FrNktWVabasrojBLOtwVuZ7ZiYtiXtmKpRNo+UsWZ1my2KlzkyR+LWcJasr4WwZFC+YQfazZkXrzMybOINZ5FBmZrPN7BIzGzOzN0yxjpnZFWb2oJn9ysyuN7O5UfedpOlmyeo0WxYvdWZK2so6fsH0s2R1ZZwtAwWzsBTMwosrmMUxU/Z2gs9id02zzl8BZwLHu/uLgTHgYzHsOzHTzpLVabYsdurMlJSVcvyCNrNkdSWdLQMFs7AUzMKLI5iZe7v/tR1uyOx24Avu/vUWz30HuNndr6k9Ph74obsvbrHuOmAdwNKlS078yvUfiKW+7uxizuwPYTbWdk332ewf+yAwP/mymoztX8bsOY+nvt/pxFXT0xNz2f3UPGYPRbuKcv/yuQztnH7GM01p1bNu6C2xb/Oa/q/Gvs1W1l30lvvc/aRUdlaT1Pi14WvvT7Tuqe2iZ9YVHY9ho+MfIIsxbGT/cnrn7Ex8P0MTfR2t1zu6kJGeZxKupr3dY3MAWDTZx+9nZHPh5pHRg2/4s2xmL49PJHNl+xmjB84RvfPfXx/7Pj77ov/JjNF4Ms9U3nHeGyKNX2ndZukoYGvD463AIjNb4O67GlesDXzXABz9ogFfMbg+pRKft/fJIUafbT+YAZiNsWDxR5i7tD/hqg722JZLyeL7M524aloBbBh6GRt/fEqkWzOdc+nJbFx/T+R64pJaPR3cOqlbefo+pizU+PWiFw346sGPpldlg11P7mJ4d+djWP/ij7Bg6YKEqzrYti2Xkcb3aDV0dGumNY+cxdYjbkm8nk5s2jnIm4aP54a+X2RTQB8H3ZrpPfNWceWeZG+VF+XWTO18+jePPvf3Q7cnG87CSutEfwMaT1wYT3n/HevkXLJmOrcsGerMlJwozPgFnZ1L1qys55Y1KtrhzDOWb2H+7P2Z1lDmzsykb2YeVlqDyg5goOHxALAHyH6euElH55I107lliVJnpmSsMOMXdHguWbMSn1vWqGjBDLI/z6zMnZl5DGaJhDIzW2xmd5jZ0bVFG4D/amb1ecl3ARs9rhPaYhJmlqxOs2XJUmempKWo4xeEmyWrq8JsGSiYhZXFrFka8hbMkpop6wNWAfWTFL4K/BS428zuJTij9F0J7Tu0ULNkdZotS5w6MyUlhRy/IOQsWV1FZstAwSys3p7x9isVUJ6CWWwn+rv76Q1/fwRY2fB4Avj72lcuRZklqxvds5dD+uczY9bMmKqSZhuOu44NAy9jE6ey4qaHsi5HSqLo4xdEmyWrG352mHn985hZgTGsHsw6aQDIizOWb2HTzsFMa1i98smDGgDK4NlVlouT/3N5omoWIs2S1Wm2LBW6Z6bIwSLNktVVaLasrmizZnmYMSvzocysZ80UyohnlqxO55alQ52ZIs+LY5asrirnljUqYjDLOpyVNZhBtoczFcqIaZasTrNlqVJnpkhMs2R1FZwtg+IFM8h+1iyLzsy0ZBXMFMqAiZF422/j3p5MT52ZB+sZ7uwK5lltT+I1uj/eMSfu7RVF/8xsrp4fRdbBDJKZNZsxY16s25vJoV2/JotgltYV/XNt/splHa2XxyvoS2DDcdex9oK3Yj84Ug0AwCu+dn5H6+XtjgcSztIjOjvxOq0r6BfZufM3F+rkfyhnA8ARA//Qdp3eh3u48IWHH3Cl/ril3QCgmTIpjQ3HXccZb7tLl8wQkUjOnb+5cIczyzpjNp0yXmRWoUxKRZ2ZIhIXBbPupR3MJnsmUwlnaXVmKpRJ6agzU0TiUsRglnU40z0zw1Mok9K6+ayrGFneowYAEYmkaMEMsp810z0zw1Eok1Jbs/AJdWaKSGQKZuEomHVHoUxKT/fMFJE4KJiFo2DWOYUyqQR1ZopIHNSZGY6CWWcUyqQy1JkpInFRMOteFsGsaJ2ZCmVSKerMFJG4FDGYZR3O1Jk5PYUyqSTdM1NE4lC0YAbZz5qVuTMzKoUyqSzdM1NE4qBgFo6C2cEUyqTS1JkpSRlnRuHuoSjhKZiFo2B2IIUyqTx1ZkqSFMyqQ52Z4SiYPU+hTITnOzMVzCQJCmbVomDWvbJ2ZnZLoUykZm3/nXz1PZ9UZ6YkQsGsWooYzLIOZ2XuzOyUQplIE3VmSlIUzKqlaMEMsp81q3pnpkKZSAvqzJSkKJhVi4JZOFUNZpFDmZmdZmabzex+M7vXzE5tsc4yM/tXM/u5md1tZv9mZi+Pum+RJKkzs/yyGr9u3H2CwlmFKJiFU8VgFimUmdlCYCNwgbsfC1wC3GpmfU2rfhh4CjjB3U8BPgn8zyj7FkmDOjPLKw/jl4JZdagzM5yqBTNz9/AvNns9cJG7v6xh2S+A97v7NxuWfQg4Gniru+83s78Gzq8NcM3bXAesA1i6dMmJX7n+A6Hri9vY/mXMnvN41mU8J2/1QP5qiquepyfmsvupecweGom0nf7lcxnauTdyPXHJWz0A6y56y33uflLS+0l6/FqydMmJn/nqRzqqpX/mcKR/S6dG9i+nd87OVPbViarWMzTRnPtb6x1dyEjPMwlX097usTkALJrs4/cz0nmvNhsZndVy+bKZvTw+EW1cnsqM0e7nrd75ptdHGr9a/ys7dxSwtWnZ1tryRh8ErgGeMLNngCeBv2i1QXe/prYuR79owFcMro9YYnwe23Ipqmd6easprnpW1P48+5aLOWb99tDbOefSk9m4/p7I9cQlb/WkLNHx66gXrfatR9zScTFpzKJs23IZqwc/mvh+OlXVelbT2SzpmkfOopv3UOJ++wZu6PtFNvvug207lh60+D3zVnHlnvBjcju9D/cktu1Wop5TZsBE07LxFtt9H3A4cIS7DwBfAL5lZjMj7l8kVerMLJVcjV86lFktRTuUCTB/9v5M91+FzsyooWwHMNC0bKC2vNGbgKvdfReAu/8LwSB3XMT9i6ROnZmlkbvxS8GsWooYzHSeWbKihrJbgWPN7KUAZnYKMAjcZmZ3mNnRtfX+HTjHzGbU1vtTYD7wcMT9i2RCnZmlkMvxS52Z1aJgFk5Zg1mkUFb75Pg64Fozuxu4CjgT6ANWAQtqq76DYBDbbGb/D/Bx4LXu/lSU/YtkSZ2ZxZb38UvBrDrUmRlOGYNZ1BP9cfcfASe3eGplwzo7gTdG3ZdI3qztvxPeBps4lRU3PZR1OdKlvI9fN+4+oXC/rCW8c+dvLlQYP2P5FjbtHMy0ht6e8VT3Vw9mSTUA6Ir+IhHpnpmSpCL9kpboihbCdc/MeCmUicREnZmSFAWzailaMIPsD2eWpTNToUwkRurMlKQomFWLglk4RQ9mCmUiMVNnpiRFnZnVktadHuKkYBaNQplIAtSZKUlSMKsOdWaGU9RgplAmkpC1/XcqmEliFMyqRcGse0UMZgplIglSZ6YkScGsWooYzLIOZ1l0ZkahUCaSgnpnph8yO+tSpGQUzKqlaMEMsp81y6IzMyyFMpGU3HzWVYwfNqnOTImdglm1KJiFU4RgplAmkqIjD3lanZmSCHVmVouCWTh5D2a5DmUjEzrUI+WjzkxJkoJZdagzM5w8B7Nch7IZY5OcfcvFbBh6WdaliMRKnZmSJAWzalEw615eg1muQxnAMeu3s+naUxXMpHTUmSlJUjCrliIGs6zDWR6DWe5DGcCKmx5SMJPS0j0zJSkKZtVStGAG2c+a5a0zsxChDJ4PZmt/+dasSxGJne6ZKUlRMKsWBbNw8hLMChPKIAhmiz43V8FMSkn3zJSkqDOzWhTMwslDMCtUKAM45IFHWfS5uZx9y8VZlyISO3VmSpKGJvqyLkFSos7McLIOZoULZRAEs2PWb1dnppSSOjMlSZoxqxYFs+5lGcwKGcrq1JkpZaXOTEmSglm1FDGYZR3OsgpmhQ5loM5MKTd1ZkpSFMyqpWjBDLKfNcuiM7PwoQzUmSnlps5MSYqCWbUomIWTZjArRSgDdWZKuakzU5KizsxqUTALJ61gFjmUmdlpZrbZzO43s3vN7NQp1ltiZjea2QNmdp+ZfSTqvpupM1PKTJ2Z8cvT+JU1BbPqUGdmOGkEs0ihzMwWAhuBC9z9WOAS4FYz62tarxf4FnC1u7/E3U8EPhFl31NRZ6aUmToz45P0+DXhxTsQoWBWLQpm3Us6mEUdNV4NPOjuPwNw99uB3wGvbFrvLcBdwIW1T6MbgNkR9z0tdWZKWakzMzaJj1+bdg7GV21KFMyqpYjBLOtwlmQwM3cP/2Kz9wEvdve1DctuAn7i7lc3LNsAHA+8BngY+BDwR+7ePPhhZuuAdQBLliw58cPv/2To+gDG+nuZv2QPi2fujbQdgLH9y5g95/HI24lL3uqB/NVU9nq2PnMYPbsc2zcW6vX9y+cytDP6/404rbvoLfe5+0lJ7yfx8WvpkhP/+5c+BcD82fsT/Jd0rnd0ISM9z3S0bv/M4YSrgZH9y+mdszPx/XSqyvV0cmHhbt4/adg9NodFk338fkby79WpjIzOOuDxhee+MdL4Nav9KtMyYKJp2TgHz8AdBnzZ3bcBmNnHgF1mNs/d9zSu6O7XANcArB440jeuvydiifDYa4/EXzXEhuOui7adLZeyYnB95Hrikrd6IH81lb2eFcDaX76VRZ+byyEPPNr168+59GTi+D9WUImOX6uOPspv6PvFARvK+hP+mkfOYusRt3S8ftKzKNu2XMbqwY8muo9uVLme1bSfJe32/ZOK376B5v9nqeqDbTuWxra5qIcvdwADTcsGassbPQHsbng82fCVOHVmSpmpMzO01Mevoh3OVGdmtRTtUCbkYxY6zsOZUUPZrcCxZvZSADM7BRgEbjOzO8zs6Np6G4F1ZnZo7fHFwG3untqcozozpczUmRlKJuNX0YIZ6DyzKlFnZjhxBbNIoczddwGvA641s7uBq4AzgT5gFbCgtt7NwP8C7jGznwMnA+dF2XcY6syUMlNnZneyHL8UzCTvFMy6F0cwi3pOGe7+I4JBqtnKpvU+Dnw86v7icMz67Wzaeiq8LfhFJlIWa/vvZO177uTsNRdzzPrtWZeTe1mOX5t2DubiF0k3btx9QuF+WUt4587fXKgwXv//VMQPPXXFu5BOTHTPTCkz3TOzGIr4y6NIv6QluiKG8KJ92GlU2VAGumemlJvumVkMm3YOFi6cKZhVi4JZeiodykCdmVJu6swsjiIGM4Wz6lAwS0flQxmoM1PKTZ2ZxVG0YAaaNauS/pnDhQtnRQtmCmU16syUMlNnZnEomEneKZglR6Gsie6ZKWWle2YWh4KZ5F0Rg1kRwplCWQvqzJQyU2dmMSiYSd4VLZhB/mfNFMqmoM5MKbN6Z6YfMjvrUmQa6syUvFMwi5dC2TTUmSlltuG46xg/bFLnmRVAEYOZwll1KJjFR6GsDXVmSpkdecjTagAoiKIFM9CsWZXonpnxUCjrQL0zc+szh+k8MykddWYWh4KZ5J2CWTQKZV3o3TmqBgApJXVmFoeCmeRdEYNZXsKZQlmX1JkpZabOzGJQMJO8K1owg3zMmimUhaDOTCkz3TOzGNSZKXmnYNY9hbKQ1JkpZaZ7ZhZHEYOZwll1KJh1R6EsAnVmSpnpnpnFUbRgBpo1qxJ1ZnZOoSwi3TNTykydmcVRxGA2NNGXdQmSIgWz9hTKYqJ7ZkpZqTOzOIoYzDRjVi1FDGZwpKIzAAAYOklEQVRphjOFshipM1PKTJ2ZxaBgJnlXtGAG6c2aKZTFTJ2ZUmbqzCwGdWZK3imYtaZQlgB1ZkqZqTOzMz6ZdQXFmzVTZ2a1KJgdTKEsIerMlDJTZ2Zntu1YmnUJhQtmoFmzKlFn5oEihzIzO83MNpvZ/WZ2r5md2mb9D5rZmJmtjrrvvFNnppRZGToz0xi/FMzCUTCrFgWzQKRQZmYLgY3ABe5+LHAJcKuZtexzNrPXAC8AHo2y36JRZ6aUVZE7M9McvxTMwlEwq5YiBrO4w1nUmbJXAw+6+88A3P124HfAK5tXNLNjgIuBiyLus5DUmSllVtDOzFTHr207lmYeznaPzcl0/2EomFVL0YIZxDtrZu4e/sVm7wNe7O5rG5bdBPzE3a9uWDYf+DbwZnffbmbbgNPdfVuLba4D1gEsWbLkxA+//5Oh64tb//K5DO3cG2kbY/29MH+cIw95OnI9Y/uXMXvO45G3E6e81aR6phd3PQ/tW8ysJ2Zg+8ZCb2PdRW+5z91Piq2oKSQ+fi1dcuIH/vkzLffd2zMe47+kc4sm+/j9jGEA5s/en0kNjXpHFzLS80xH6/bPHE64GhjZv5zeOTsT30+nqlxPpxcW7uY9lLTdY3P4679cG2n8mhWxBgMmmpaN0zADZ2YGfAW4wt23t9ugu18DXAOweuBI37j+noglxuecS08mjnr2veRwfn/BXjYcd12k7Ty25VJWDK6PXE+c8laT6ple3PWsANb+8q3YD/pZcdNDsW03IYmOXwNrjvIr90z9ktUrnwxRcjRvGj6eG/p+8dzjrG++vOaRs9h6xC1dvSbJmZRtWy5j9eBHE9t+t6pcz2o6myUN8x7Ks6iHL3cAA03LBmrL6w4Fjgf+0czuMrO7CM7LuNnMzou4/0JSZ6aUWYE6MzMdv7I+lAk6z0zyrYidmVFFDWW3Asea2UsBzOwUYBC4zczuMLOj3X23ux/p7qfWvwjO2zjb3b8ccf+Fpc5MKbOCdGZmPn4pmIWjYFYtVQpmkUKZu+8CXgdca2Z3A1cBZwJ9wCpgQeQKS06dmVJW9WCW187MvIxfCmbhKJhVS1WCWdRzynD3HwEnt3hq5TSvWR11v2Wy4qaH2MSp8LbgF5lIWaztv5O1Z93J2VzMwLfHOeSBfF0NJy/jVz2YZXGeWd2mnYOZn2PWrRt3n1CZX9YSBLOyh3Fd0T8ndM9MKTPdM7MzWc+a6Z6ZkndlD+EKZTmie2ZKmememZ3JOphB8Q5n6p6Z1VLmYKZQljPqzJQyK1BnZqYUzMJRMKuOsnZmKpTlkDozpcwK0pmZOQWzcBTMqiWNiwqnSaEsx9SZKWWV987MvFAwC0fBrFrKNGOmUJZzumemlNXa/juLes/MVOXhnpkKZpJ3ZQlmCmUFoM5MKTN1ZnYmD8GsaOFMwaxayhDMFMoKQp2ZUmbqzOxM1sEMijdrps7Mail6MFMoKxB1ZkqZ1TszZXoKZuEomFVHkTszFcoKprEz8+mJuVmXIxIr3dGiMwpm4SiYVUsRg5lCWUEds347u5+apwYAkYpSMAtHwaxaihbMFMoKbPbQiDozRSpMnZnhKJhVS5GCmUJZwakzU0TyEMyKFs4UzKqlKMFMoawE1JkpkkNuqe4u62AGxZs1U2dmtRQhmCmUlYQ6M0Xyp/fhnlT3p2AWjoJZdeS9M1OhrER0z0yR/FEwKwYFs2rJazBTKCsh3TNTJF8UzIphaKIv6xIkRXkMZgplJaV7ZorkSxbBLOtwVsRgphmzaslbMFMoKzF1ZorkS+/DPZWbNVNnpuRdnoKZQlnJqTNTJH+qFsygeLNm6syslrwEM4WyClBnpkj+KJgVg4JZdeShM1OhrCLUmSmSP2kHs5HRWanurxUFM8m7LINZ5FBmZqeZ2WYzu9/M7jWzU1uss8zMvmhmvzazu83sp2b20qj7lu6pM1PkeXkYvzRjVgwKZtWSVTCLFMrMbCGwEbjA3Y8FLgFuNbPmvuITgO+5+x+4+ynALcCVUfYt4akzUyRf45c6M4tBwaxasghmUWfKXg086O4/A3D324HfAa9sXMndv+PuGxsW/Q7Ifh69wtSZKZKv8auKnZm7x+YULpwpmFVL2sHM3D38i83eB7zY3dc2LLsJ+Im7Xz3Fa5YBdwDnu/uPWzy/DlgHsGTJkhM//P5Phq4vbv3L5zK0c2/WZTwnjnr8kNmMHzbJkYc8HUtNY/uXMXvO47FsKw6qZ3p5qwfgzFdfeJ+7n5T0fpIfv5aeePmnPxuqtsmeyVCva2fZzF4enxg5aHlvz3gi+2tn0WQfv58xDMD82fszqaFR7+hCRnqe6Xj9/pnDCVYDI/uX0ztnZ6L76Ebe6oH0aur0wsJv/PO3Rxq/on7aM2Ciadk4U8zAmdli4NvA5a0GNAB3vwa4BmD1wJG+cf09EUuMzzmXnkwZ69n3ksN5+MxZ3HzWVZG39diWS1kxuD7yduKieqaXt3pSluj4NXDUGv/0bx4NXdzIwGjo107lPfNWceWe7S2fW73yydj3186bho/nhr5fPPf4jOVbUq+h0ZpHzmLrEbd09ZokZ1K2bbmM1YMfTWz73cpbPZBeTatrfyY9Uxr18OUOYKBp2UBt+QHM7AXAD4Er3f36iPuVGDV2ZopUSK7Hr6odygSdZyb5l/ThzKih7Fbg2HonkpmdAgwCt5nZHWZ2dG35KoIB7Qp3vyHiPiUhx6zfzluufLcaAKQqcj9+KZgVg4JZtSQZzCKFMnffBbwOuNbM7gauAs4E+oBVwILaqlcCy4D3mtldta+W0/+SLXVmSlUUZfxSZ2YxKJhVS1LBLHIHkbv/CDi5xVMrG9Y5N+p+JD0rbnqITZzK9191DBuOuy7rckQSU5Txqx7MkjjPbCrbdizN5Dyzunowy/o8s27cuPuEzK8IL+k5d/7m2MO4rugvLememSL5o8OZ+ad7ZlZL3CFcoUympHtmiuSPglkxKJhVR5z3zFQok2mpM1MkfxTMikHBrFriCGYKZdIRdWaK5IuCWTEomEk3FMqkY+rMFMkXdWYWg4KZdEqhTLqie2aKdMbC38GuK1W8Z+amnYOFC2cKZtIJhTLpmjozRTpz6Hbn0O3ppLOqBTMo3qyZOjOlHYUyCUWdmSKdUzBLTtGCGWjWTKamUCahqTNTpHMKZslRMJOyUCiTyNSZKdIZBbPkKJhJGSiUSSzqDQBPT8zNuhSRXCtzMMs6nCmYSdEplElsVtz0ELufmqcGAJE20gxmVZs1U2emFJlCmcRq9tCIOjNFOqDOzGQVMZgpnIlCmcROnZkinVMwS07Rghlo1qzqFMokEerMFOlcWsFsxmi6Q76CWTgKZtU1K+sC8uC2N3+J0b7htut9dwh4e/vt9Qz38YqvnR+9sBI4Zv123rL13ZzxtrtY239n1uVIg8nxCXpmXc3k+AQzZs3MupzKO3S78+wqC/XarZMfZIJn2673zn/vbHszZszjiIF/CFVLs207lrJ65ZOxbCusTTsHOWP5lkxr6NaNu0/gpKyLyLmJ8Qlmz/w0E+MTzCzJGKaZMugokGW5vaLTPTPzad/QbmbYb9k3tDvrUqQm7IxZJ4GsG5OTe2Ldnjozwxma6Mu6hFzbM7QHs9+yZyje92uWFMokFbpnZr5Mjk8wumcvZs7onr1Mjk9kXZLUpHUoMwt5CGZFC2c6lNnaxPgEw88OY+YMPzvMREnGMIUySY3umZkf+4Z2Q/13v6PZspxJszMzbVkHMyjerJk6Mw+2Z2jPAWNYWWbLFMokVerMzF59lqyRZsvyScEsOUULZqBZs7r6LFmjssyWKZRJ6tSZma0DZsnqNFuWWwpmyVEwK6YDZsnqSjJbplAmmdE9M9PXapasTrNl+VXWYDYymv0FABTMiqXVLFldGWbLIocyMzvNzDab2f1mdq+ZndpiHTOzK8zsQTP7lZldb2a6SaKoMzNlLWfJ6io4W1ak8auswUydmeFUNZi1nCWrK8FsWaRQZmYLgY3ABe5+LHAJcKuZNffx/hVwJnC8u78YGAM+FmXfUh7qzEzHdLNkdVWaLSvi+JVmMKvaHQA27Rxk99icTGvoVtWC2XSzZHVFny2LOlP2auBBd/8ZgLvfDvwOeGXTeq8Hvuju+2qPrwbeGHHfUiLqzEzetLNkddWaLSvk+KV7ZiaraLNmVerMnHaWrK7gs2VRD+gfBWxtWra1tny69bYCi8xsgbvvalzRzNYB6wCWLFnCOe8/OWKJ7X13KP5tnnNp8nX3L5+byn66EbUm3zmbn+77BGsWPhFLPWP7l/HYlktj2VYcsqtnF3Nmfwjr4ILxI7tH2fX03wDzE6+qtQvT2lHi49f5f3J4vBU3mewJfqCdXqm/Gxe+8PnaJ3sm498BsGxmL++Zt+rAhc+sordnPJH9tbNoso83DR8Pvz2e+bP3Z1JDo97Rhax55KyO1v05Z9E/M9kLl4/sX862LZcluo+p7aJn1hUdjWF7d40x9PQ7yGYMuyjSq6OGMgOa5wnHOXgGrnm9+v+4g2bq3P0a4BqA1QNH+sb190QssQMd3DqpW2nUfc6lJ6eyn27EVdODl67i5rOuirydx7ZcyorB9ZG3E5es6tn75BCjz451tK7ZGAsWf4S5S/sTripziY5fq1Yd5V/66aPxVDqNsLdmaufTvzmw9pGB0dj38Z55q7hyz/aWz2Vxa6Y3DR/PDX2/eO5x1rdmWvPIWWw94pauXnPu/M0JVQPbtlzG6sGPJrb96ex6chfDuzsfw/oXf4QFSxckXFX8oh6+3AEMNC0bqC2fbr0BYA/wTMT9S0mpMzM+nZxL1qwi55YlPn4t2DoSpb6O6FBmcop2KBPKeZ5ZJ+eSNSvquWVRQ9mtwLFm9lIAMzsFGARuM7M7zOzo2nobgP9qZvX/1e8CNrp7OduJJBbqzIxHR+eSNavGuWWpjF9pBLO0ZBHMsg5nCmbZ6+hcsmYFPbcs0uFLd99lZq8DrjUzJ5jWPxPoA1YB9bnDrwIvBO42s3HgV8A7o+xbqmHFTQ+xiVP5/quOYcNx12VdTuGEmSWrG92zl0P65zNj1syYq8qHNMevejDbtaY3nuIzVA9mSRzOnMq2HUszOZxZVw9mWR/O7MaNu09I9FBmWsLMktUNPzvMvP55zCzQGBb5yn3u/iOg1ZndKxvWmQD+vvYl0pUVNz3EvgcPZ+0Fb1Uw61KoWbK62mxZmc8tS3v8WrB1pBTBDIJwVqVgBkE4K1owg2TPM0taqFmyutpsWZHOLdMV/aUQdM/M7kWZJauryLllqdLhzPCyPpQJOpyZpiizZHVFO7dMoUwKQ/fM7E6kWbK6apxbljoFs/AUzMIpYjCLNEtWV7BzyxTKpHDUmdleHLNkdZotS4aCWXgKZuEUKZjFMUtWV6TZMoUyKSR1Zk4vllmyOs2WJUbBLDx1ZoZTlGAWyyxZXYFmyxTKgJ7h5lvd5Wt70prumTm1iZF4T8COe3vyvAVbRyKHs1kTh8ZUTWAm4bbX+3BP5WbNNu0cLFw4K0IwG90f75gT9/aSErn7sgxe8bXzO1ovj1fQrzp1ZrY2f+WyjtbL2x0PqixKZ+Zxj13e0Xrn/8nhXPXwY6H20Q11ZuZf3jszlx7RWdjO8i4DSdBMmRSeOjOlLHQHgPCynjEDHc6U6BTKpBTUmSlloWAWnoJZOApm+aFQJqWizkwpAwWz8BTMwlEwyweFMikddWZKGSiYhafOzHAUzLKnUCalVA9mD+1bnHUpIqHF0ZnZTprBrGqzZurMlG4plElprbjpIWY9MUOXzJDCSyOYlXnWLGtFDGYKZ9lQKJNSs31j6syUUtDhzPAUzMJRMEufQpmUnjozpSwUzMJTMAtHwSxdCmVSGerMlDIoUzCbMZruryAFs3AUzNKjUCaVos5MKYMyBTN1ZhaDglk6FMqkcnTPTEmDebKhRp2Z0eQhmBUtnA1N6L7OSVMok0pacdNDLPrcXAUzSVTPlh2J70OdmeFlHcygeLNm6sxMlkKZVJbumSlp6NmyI/FwpsOZ4SmYhaNglgyFMqk0dWZKWhTMOqdgVgwKZvFTKBNBnZmSDgWzzimYFYOCWbwUykRq1JkpaVAw61zawWxkdFbm4UzBrNoihTIzO83MNpvZ/WZ2r5mdOsV6y8zsi2b2azO728x+amYvjbJvkSSoM7NashrD0ghm6swMLw/BrGjhTMEsHqFDmZktBDYCF7j7scAlwK1m1qpn9gTge+7+B+5+CnALcGXYfYskSZ2Z1ZD1GKbOzO5ULZgB7B6bk3UJXVFnZnRRZspeDTzo7j8DcPfbgd8Br2xe0d2/4+4bGxb9DpgVYd8iiVJnZiVkPoapM7M7VQxmRZsxA82aRWHe5gKHZtYD/KTFU98BXujuaxvWvQn4ibtfPc32lgF3AOe7+49bPL8OWFd7+BLggXb/iBQtAZ7KuogGeasH8leT6ple3uoBOMbdD41rY2mOYTkfvyB/P2/VMz3V017eaoo0frX9pOfuo8BB51mY2d8BE02Lx5lm9s3MFgPfBi5vFchq+7sGuKa2/r3uflK7GtOietrLW02qZ3p5qweCmuLcXppjWJ7HL8hfTapneqqnvbzVFHX8inL4cgcw0LRsoLb8IGb2AuCHwJXufn2E/YqIxEFjmIjkSpRQditwbL0DycxOAQaBTWa22MzuMLOja8+tIhjMrnD3G6IWLSISA41hIpIroU9UdfddZvY64Fozc4Jp/zPd/RkzOwJYBSyorX4lsAx4r5m9t7ZsxN1Pa7Oba8LWlxDV017ealI908tbPZBSTSmMYZX93nZB9UxP9bSXt5oi1dP2RH8RERERSZ6u6C8iIiKSAwplIiIiIjmgUCYiIiKSA7kKZXm4l2YnNVjgCjN70Mx+ZWbXm9ncOPYfsp7U7i3a6c+oYf0PmtmYma1Oop5uajKzJWZ2o5k9YGb3mdlHsqqn9jP7VzP7ee1n9m9m9vKE6pltZpfUfg5vmGKdNN/TndRTuPvlavwKXU+qP+u8jWEav9rWk6vxq4uawr2v3T0XX8BC4Gngj2qPTwceB/parPvnwDkNj98DfD+tGoDzgPuAQ2qPrwM+m9X3JKnvR5SfUe351wBfALYBq7N83wC9wF3AnzQsW5xhPV8C/pnnm23OAR5N6Hv0jtp74qfAG6ZYJ5X3dBf1pPKeTvvnnuS/TeNXvD+n2vOJjmEavzqqKVfjVxc1hXpf52mmLPP70HVRw+uBL7r7vtrjq4E3xrD/UPUk+P0IVQ+AmR0DXAxclEAdYWp6C8GgdmHt098GYHaG9TxKMAD21h4vrS2Lnbt/3t2v5OCr1zdK6z3dUT0pvqfjovErZD0p/6zzNoZp/Gojb+NXpzWFfV+nPsjZ9Peh29q0bCtwVJvtLQM+BJwfQ3lHdVhD83pbgUVmtsDdd8VQR7f1PCfm70eoesxsPsEnqTe7+4iZJVBKdzUBfwocT/DJ92GC79HXaDEYp1TPBwmuZ/OEmT0DPAn8Rcy1dCOt93TXEn5Pd1uLxq/463lOCj/rvI1hGr/ikdvxC7p7X6ceyjzle2l2yTqsoXm98dqfcc88dlpPsHL834+u67Fg9PoKwZXPtydQQ9c11RwGfNndtwGY2ceAXWY2z933ZFDP+4DDgSM8uIjp24Fvmdl/cPfpPhEmJa33dFdSeE93ReNXIvUEK6fzs87bGKbxKx65HL+g+/d15gU3yMN96DqtoXm9AWAP8ExMdXRbT1r35euknkMJPtH9o5ndZWZ3AS8Abjaz8zKqCeAJYHfD48mGryzqeRNwdf1TnLv/C8Egd1zM9XQqrfd0x1J6T8dF41f4etL8WedtDNP4FY/cjV8Q8n2d1Ilw3X4R3M7kKeCltcenAEMEx60XA3cAR9eeWwX8CnhtSjU07/+tBIcwemqPPwt8JcXvSSrfj7D1tHjdNpI70b/T79HZwD3AobXHf08yzRCd1rMR+CIwo/b4T4G9wJIEf363UzspNav3dBf1pPKeTuHnrvErJ+NXNzW1eF0iY5jGr65qy9X41UFNod7XuTlx1tO5l2aoGoC+pv1/FXghcLeZjRN8498ZZd8R60nk+xGhntR0WpO732xmLwTuMbN9wHaCjp1M6iHo3vkUsNnMRmrLXuvuT8Vd0xQyeU93UU8q7+m4aPyKVE9qP+u8jWEav0LL2/jVqqZQ72vd+1JEREQkB/J0TpmIiIhIZSmUiYiIiOSAQpmIiIhIDiiUiYiIiOSAQpmIiIhIDiiUiYiIiOSAQpmIiIhIDiiUiYiIiOTA/w/I266Sc3h/QgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"활성화 함수: 헤비사이드\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"활성화 함수: 시그모이드\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST를 위한 FNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "주의: `tf.examples.tutorials.mnist`은 삭제될 예정이므로 대신 `tf.keras.datasets.mnist`를 사용하겠습니다. `tf.contrib.learn` API는 `tf.estimator`와 `tf.feature_column`로 옮겨졌고 상당히 많이 바뀌었습니다. 특히 `infer_real_valued_columns_from_input()` 함수와 `SKCompat` 클래스가 없습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0\n",
    "y_train = y_train.astype(np.int32)\n",
    "y_test = y_test.astype(np.int32)\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:]\n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Estimator API를 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpenr8wre6\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmpenr8wre6', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fadb0686ef0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpenr8wre6/model.ckpt.\n",
      "INFO:tensorflow:loss = 119.99919, step = 0\n",
      "INFO:tensorflow:global_step/sec: 595.58\n",
      "INFO:tensorflow:loss = 25.30537, step = 100 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.33\n",
      "INFO:tensorflow:loss = 17.72889, step = 200 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 660.178\n",
      "INFO:tensorflow:loss = 8.094297, step = 300 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 684.089\n",
      "INFO:tensorflow:loss = 11.453514, step = 400 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.726\n",
      "INFO:tensorflow:loss = 6.225746, step = 500 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.083\n",
      "INFO:tensorflow:loss = 4.65758, step = 600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.379\n",
      "INFO:tensorflow:loss = 18.605453, step = 700 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 655.74\n",
      "INFO:tensorflow:loss = 5.270692, step = 800 (0.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.019\n",
      "INFO:tensorflow:loss = 6.3098373, step = 900 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.652\n",
      "INFO:tensorflow:loss = 3.4994135, step = 1000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.94\n",
      "INFO:tensorflow:loss = 7.370407, step = 1100 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 688.764\n",
      "INFO:tensorflow:loss = 5.695126, step = 1200 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 718.536\n",
      "INFO:tensorflow:loss = 6.970705, step = 1300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.423\n",
      "INFO:tensorflow:loss = 10.361841, step = 1400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.52\n",
      "INFO:tensorflow:loss = 2.355143, step = 1500 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 785.184\n",
      "INFO:tensorflow:loss = 4.8159637, step = 1600 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 680.541\n",
      "INFO:tensorflow:loss = 0.9250039, step = 1700 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 551.339\n",
      "INFO:tensorflow:loss = 10.602991, step = 1800 (0.182 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.725\n",
      "INFO:tensorflow:loss = 3.7039974, step = 1900 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 645.019\n",
      "INFO:tensorflow:loss = 8.637078, step = 2000 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 626.024\n",
      "INFO:tensorflow:loss = 6.450728, step = 2100 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.047\n",
      "INFO:tensorflow:loss = 12.033913, step = 2200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.537\n",
      "INFO:tensorflow:loss = 2.2622938, step = 2300 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 670.679\n",
      "INFO:tensorflow:loss = 4.6523166, step = 2400 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.025\n",
      "INFO:tensorflow:loss = 7.9452434, step = 2500 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.293\n",
      "INFO:tensorflow:loss = 0.90992653, step = 2600 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 671.676\n",
      "INFO:tensorflow:loss = 6.538801, step = 2700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.182\n",
      "INFO:tensorflow:loss = 6.7619023, step = 2800 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.016\n",
      "INFO:tensorflow:loss = 3.85844, step = 2900 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.002\n",
      "INFO:tensorflow:loss = 6.7363853, step = 3000 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.567\n",
      "INFO:tensorflow:loss = 1.7025919, step = 3100 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 688.566\n",
      "INFO:tensorflow:loss = 5.8390517, step = 3200 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.924\n",
      "INFO:tensorflow:loss = 2.9001834, step = 3300 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 643.728\n",
      "INFO:tensorflow:loss = 3.1886394, step = 3400 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 682.034\n",
      "INFO:tensorflow:loss = 2.7842398, step = 3500 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 663.933\n",
      "INFO:tensorflow:loss = 1.900357, step = 3600 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 789.021\n",
      "INFO:tensorflow:loss = 0.6681113, step = 3700 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.614\n",
      "INFO:tensorflow:loss = 0.079646066, step = 3800 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 648.854\n",
      "INFO:tensorflow:loss = 0.3074991, step = 3900 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 641.604\n",
      "INFO:tensorflow:loss = 14.438376, step = 4000 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 675.755\n",
      "INFO:tensorflow:loss = 0.46155563, step = 4100 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.269\n",
      "INFO:tensorflow:loss = 2.7854247, step = 4200 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 665.538\n",
      "INFO:tensorflow:loss = 0.6228178, step = 4300 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 661.948\n",
      "INFO:tensorflow:loss = 1.5486633, step = 4400 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 678.159\n",
      "INFO:tensorflow:loss = 0.06805645, step = 4500 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.048\n",
      "INFO:tensorflow:loss = 1.6644863, step = 4600 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.093\n",
      "INFO:tensorflow:loss = 0.7109455, step = 4700 (0.168 sec)\n",
      "INFO:tensorflow:global_step/sec: 601.031\n",
      "INFO:tensorflow:loss = 0.45003325, step = 4800 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.408\n",
      "INFO:tensorflow:loss = 0.83514655, step = 4900 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 692.79\n",
      "INFO:tensorflow:loss = 1.513859, step = 5000 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 676.147\n",
      "INFO:tensorflow:loss = 0.13689423, step = 5100 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.596\n",
      "INFO:tensorflow:loss = 0.36891767, step = 5200 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 740.482\n",
      "INFO:tensorflow:loss = 0.478229, step = 5300 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.08\n",
      "INFO:tensorflow:loss = 0.5095083, step = 5400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.091\n",
      "INFO:tensorflow:loss = 4.1548905, step = 5500 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.589\n",
      "INFO:tensorflow:loss = 0.43653432, step = 5600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.129\n",
      "INFO:tensorflow:loss = 0.67023534, step = 5700 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.049\n",
      "INFO:tensorflow:loss = 0.647523, step = 5800 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 795.468\n",
      "INFO:tensorflow:loss = 0.09369692, step = 5900 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 783.111\n",
      "INFO:tensorflow:loss = 0.44570932, step = 6000 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.271\n",
      "INFO:tensorflow:loss = 1.6947678, step = 6100 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.163\n",
      "INFO:tensorflow:loss = 1.5905691, step = 6200 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 771.803\n",
      "INFO:tensorflow:loss = 2.0049686, step = 6300 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 785.546\n",
      "INFO:tensorflow:loss = 0.20519589, step = 6400 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.503\n",
      "INFO:tensorflow:loss = 1.7045006, step = 6500 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 707.078\n",
      "INFO:tensorflow:loss = 1.706248, step = 6600 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.897\n",
      "INFO:tensorflow:loss = 0.18770057, step = 6700 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.567\n",
      "INFO:tensorflow:loss = 0.8759486, step = 6800 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.265\n",
      "INFO:tensorflow:loss = 0.15285236, step = 6900 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.004\n",
      "INFO:tensorflow:loss = 1.386205, step = 7000 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 753.287\n",
      "INFO:tensorflow:loss = 1.3452835, step = 7100 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.285\n",
      "INFO:tensorflow:loss = 0.21504202, step = 7200 (0.140 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 682.471\n",
      "INFO:tensorflow:loss = 0.9096737, step = 7300 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 780.481\n",
      "INFO:tensorflow:loss = 1.0315107, step = 7400 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.535\n",
      "INFO:tensorflow:loss = 0.20635258, step = 7500 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 734\n",
      "INFO:tensorflow:loss = 0.22424649, step = 7600 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 674.58\n",
      "INFO:tensorflow:loss = 0.41415858, step = 7700 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.027\n",
      "INFO:tensorflow:loss = 0.21832135, step = 7800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.591\n",
      "INFO:tensorflow:loss = 1.5665843, step = 7900 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.988\n",
      "INFO:tensorflow:loss = 0.109256245, step = 8000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.765\n",
      "INFO:tensorflow:loss = 0.39396834, step = 8100 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 675.48\n",
      "INFO:tensorflow:loss = 0.39230296, step = 8200 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 702.005\n",
      "INFO:tensorflow:loss = 0.26779386, step = 8300 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.404\n",
      "INFO:tensorflow:loss = 0.037142925, step = 8400 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 772.978\n",
      "INFO:tensorflow:loss = 0.16765824, step = 8500 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 718.028\n",
      "INFO:tensorflow:loss = 0.27566725, step = 8600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.434\n",
      "INFO:tensorflow:loss = 0.99648494, step = 8700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 740.505\n",
      "INFO:tensorflow:loss = 0.41392073, step = 8800 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 712.962\n",
      "INFO:tensorflow:loss = 0.4626464, step = 8900 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 729.972\n",
      "INFO:tensorflow:loss = 0.12112476, step = 9000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 726.188\n",
      "INFO:tensorflow:loss = 0.123770244, step = 9100 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.267\n",
      "INFO:tensorflow:loss = 0.19609205, step = 9200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.468\n",
      "INFO:tensorflow:loss = 0.18060449, step = 9300 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 676.61\n",
      "INFO:tensorflow:loss = 0.3009936, step = 9400 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 782.571\n",
      "INFO:tensorflow:loss = 0.17434198, step = 9500 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.351\n",
      "INFO:tensorflow:loss = 0.4527591, step = 9600 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 801.553\n",
      "INFO:tensorflow:loss = 0.032135107, step = 9700 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 689.654\n",
      "INFO:tensorflow:loss = 0.23623474, step = 9800 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 670.93\n",
      "INFO:tensorflow:loss = 0.16658491, step = 9900 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 821.449\n",
      "INFO:tensorflow:loss = 0.385925, step = 10000 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 653.083\n",
      "INFO:tensorflow:loss = 0.04934814, step = 10100 (0.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.155\n",
      "INFO:tensorflow:loss = 0.10290457, step = 10200 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.479\n",
      "INFO:tensorflow:loss = 0.035794564, step = 10300 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.723\n",
      "INFO:tensorflow:loss = 0.04308272, step = 10400 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 675.025\n",
      "INFO:tensorflow:loss = 0.1811445, step = 10500 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.621\n",
      "INFO:tensorflow:loss = 0.039043777, step = 10600 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.904\n",
      "INFO:tensorflow:loss = 0.13285968, step = 10700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 678.33\n",
      "INFO:tensorflow:loss = 0.10310352, step = 10800 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 810.198\n",
      "INFO:tensorflow:loss = 0.09150319, step = 10900 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.327\n",
      "INFO:tensorflow:loss = 0.08905714, step = 11000 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 709.772\n",
      "INFO:tensorflow:loss = 0.1375821, step = 11100 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.151\n",
      "INFO:tensorflow:loss = 0.07634531, step = 11200 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 637.229\n",
      "INFO:tensorflow:loss = 0.028460909, step = 11300 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.447\n",
      "INFO:tensorflow:loss = 0.09573003, step = 11400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.825\n",
      "INFO:tensorflow:loss = 0.35402524, step = 11500 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.687\n",
      "INFO:tensorflow:loss = 0.09644293, step = 11600 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.66\n",
      "INFO:tensorflow:loss = 0.018558478, step = 11700 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 836.607\n",
      "INFO:tensorflow:loss = 0.13224041, step = 11800 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 776.56\n",
      "INFO:tensorflow:loss = 0.09854912, step = 11900 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.958\n",
      "INFO:tensorflow:loss = 0.117844254, step = 12000 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.275\n",
      "INFO:tensorflow:loss = 0.20646203, step = 12100 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.173\n",
      "INFO:tensorflow:loss = 0.065734796, step = 12200 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.624\n",
      "INFO:tensorflow:loss = 0.059990425, step = 12300 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 778.767\n",
      "INFO:tensorflow:loss = 0.006515979, step = 12400 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.234\n",
      "INFO:tensorflow:loss = 0.84475434, step = 12500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 683.403\n",
      "INFO:tensorflow:loss = 0.019861495, step = 12600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 742.084\n",
      "INFO:tensorflow:loss = 0.86694646, step = 12700 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.759\n",
      "INFO:tensorflow:loss = 2.0885234, step = 12800 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.857\n",
      "INFO:tensorflow:loss = 0.05615161, step = 12900 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.037\n",
      "INFO:tensorflow:loss = 0.08216704, step = 13000 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 660.738\n",
      "INFO:tensorflow:loss = 0.27249014, step = 13100 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.73\n",
      "INFO:tensorflow:loss = 0.059483968, step = 13200 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 685.956\n",
      "INFO:tensorflow:loss = 0.03802904, step = 13300 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 596.365\n",
      "INFO:tensorflow:loss = 0.19731334, step = 13400 (0.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 703.132\n",
      "INFO:tensorflow:loss = 0.0059482413, step = 13500 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.061\n",
      "INFO:tensorflow:loss = 0.21736607, step = 13600 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 681.464\n",
      "INFO:tensorflow:loss = 0.088271424, step = 13700 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.555\n",
      "INFO:tensorflow:loss = 0.04580865, step = 13800 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 693.06\n",
      "INFO:tensorflow:loss = 0.122456014, step = 13900 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.958\n",
      "INFO:tensorflow:loss = 0.15052801, step = 14000 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 673.474\n",
      "INFO:tensorflow:loss = 0.025914695, step = 14100 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 752.748\n",
      "INFO:tensorflow:loss = 0.053208873, step = 14200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 728.193\n",
      "INFO:tensorflow:loss = 0.060223866, step = 14300 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.841\n",
      "INFO:tensorflow:loss = 0.051761948, step = 14400 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 659.322\n",
      "INFO:tensorflow:loss = 0.009877177, step = 14500 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.649\n",
      "INFO:tensorflow:loss = 0.098193645, step = 14600 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 697.427\n",
      "INFO:tensorflow:loss = 0.038370922, step = 14700 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 660.384\n",
      "INFO:tensorflow:loss = 0.04959974, step = 14800 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 593.871\n",
      "INFO:tensorflow:loss = 0.005264436, step = 14900 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.731\n",
      "INFO:tensorflow:loss = 0.37411672, step = 15000 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 802.229\n",
      "INFO:tensorflow:loss = 0.073860556, step = 15100 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.819\n",
      "INFO:tensorflow:loss = 0.10173142, step = 15200 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.22\n",
      "INFO:tensorflow:loss = 0.3613294, step = 15300 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.24135225, step = 15400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.358\n",
      "INFO:tensorflow:loss = 0.039711084, step = 15500 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.942\n",
      "INFO:tensorflow:loss = 0.06568477, step = 15600 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 743.444\n",
      "INFO:tensorflow:loss = 0.12381785, step = 15700 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 698.785\n",
      "INFO:tensorflow:loss = 0.07432026, step = 15800 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.083\n",
      "INFO:tensorflow:loss = 0.0028906446, step = 15900 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.666\n",
      "INFO:tensorflow:loss = 0.14015041, step = 16000 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.565\n",
      "INFO:tensorflow:loss = 0.14721057, step = 16100 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.868\n",
      "INFO:tensorflow:loss = 0.08058843, step = 16200 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.971\n",
      "INFO:tensorflow:loss = 0.042667102, step = 16300 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.326\n",
      "INFO:tensorflow:loss = 0.03715862, step = 16400 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.483\n",
      "INFO:tensorflow:loss = 0.031644855, step = 16500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 675.657\n",
      "INFO:tensorflow:loss = 0.06807215, step = 16600 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.592\n",
      "INFO:tensorflow:loss = 0.014378089, step = 16700 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.843\n",
      "INFO:tensorflow:loss = 0.045214668, step = 16800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 630.089\n",
      "INFO:tensorflow:loss = 0.022370774, step = 16900 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.79\n",
      "INFO:tensorflow:loss = 0.17216605, step = 17000 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 761.262\n",
      "INFO:tensorflow:loss = 0.10270027, step = 17100 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.107\n",
      "INFO:tensorflow:loss = 0.024180472, step = 17200 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 704.61\n",
      "INFO:tensorflow:loss = 0.40556854, step = 17300 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.893\n",
      "INFO:tensorflow:loss = 0.06243583, step = 17400 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.259\n",
      "INFO:tensorflow:loss = 0.029030599, step = 17500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.902\n",
      "INFO:tensorflow:loss = 0.24098389, step = 17600 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 718.83\n",
      "INFO:tensorflow:loss = 0.02992541, step = 17700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.237\n",
      "INFO:tensorflow:loss = 0.014920887, step = 17800 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.125\n",
      "INFO:tensorflow:loss = 0.050719894, step = 17900 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 819.816\n",
      "INFO:tensorflow:loss = 0.07073748, step = 18000 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.903\n",
      "INFO:tensorflow:loss = 0.04527495, step = 18100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.43\n",
      "INFO:tensorflow:loss = 0.017566409, step = 18200 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 703.49\n",
      "INFO:tensorflow:loss = 0.021974105, step = 18300 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.378\n",
      "INFO:tensorflow:loss = 0.045856465, step = 18400 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.984\n",
      "INFO:tensorflow:loss = 0.012996869, step = 18500 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 698.84\n",
      "INFO:tensorflow:loss = 0.124543935, step = 18600 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.507\n",
      "INFO:tensorflow:loss = 0.03451944, step = 18700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 670.899\n",
      "INFO:tensorflow:loss = 0.022585992, step = 18800 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 797.639\n",
      "INFO:tensorflow:loss = 0.079437256, step = 18900 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.109\n",
      "INFO:tensorflow:loss = 0.006705559, step = 19000 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 635.02\n",
      "INFO:tensorflow:loss = 0.060073216, step = 19100 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.466\n",
      "INFO:tensorflow:loss = 0.022254843, step = 19200 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 726.77\n",
      "INFO:tensorflow:loss = 0.0020687946, step = 19300 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.007\n",
      "INFO:tensorflow:loss = 0.0014081515, step = 19400 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 727.56\n",
      "INFO:tensorflow:loss = 0.025616948, step = 19500 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 639.869\n",
      "INFO:tensorflow:loss = 0.015866615, step = 19600 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 771.806\n",
      "INFO:tensorflow:loss = 0.020616727, step = 19700 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.218\n",
      "INFO:tensorflow:loss = 0.102783024, step = 19800 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.856\n",
      "INFO:tensorflow:loss = 0.025931489, step = 19900 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 693.511\n",
      "INFO:tensorflow:loss = 0.016829671, step = 20000 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.372\n",
      "INFO:tensorflow:loss = 0.033985313, step = 20100 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.583\n",
      "INFO:tensorflow:loss = 0.021627681, step = 20200 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 791.142\n",
      "INFO:tensorflow:loss = 0.018013205, step = 20300 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.614\n",
      "INFO:tensorflow:loss = 0.004799627, step = 20400 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.936\n",
      "INFO:tensorflow:loss = 0.026493322, step = 20500 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.778\n",
      "INFO:tensorflow:loss = 0.0224808, step = 20600 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.382\n",
      "INFO:tensorflow:loss = 0.007879016, step = 20700 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 667.268\n",
      "INFO:tensorflow:loss = 0.08518717, step = 20800 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 722.222\n",
      "INFO:tensorflow:loss = 0.050738066, step = 20900 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.866\n",
      "INFO:tensorflow:loss = 0.061514728, step = 21000 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 683.663\n",
      "INFO:tensorflow:loss = 0.0019400625, step = 21100 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 794.358\n",
      "INFO:tensorflow:loss = 0.06887854, step = 21200 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.291\n",
      "INFO:tensorflow:loss = 0.05445952, step = 21300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 633.455\n",
      "INFO:tensorflow:loss = 0.0042447494, step = 21400 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.401\n",
      "INFO:tensorflow:loss = 0.020919558, step = 21500 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.551\n",
      "INFO:tensorflow:loss = 0.015569954, step = 21600 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.57\n",
      "INFO:tensorflow:loss = 0.031518906, step = 21700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.985\n",
      "INFO:tensorflow:loss = 0.016381992, step = 21800 (0.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 657.575\n",
      "INFO:tensorflow:loss = 0.03825447, step = 21900 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 688.681\n",
      "INFO:tensorflow:loss = 0.09139495, step = 22000 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.844\n",
      "INFO:tensorflow:loss = 0.006838148, step = 22100 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 695.164\n",
      "INFO:tensorflow:loss = 0.02163041, step = 22200 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.496\n",
      "INFO:tensorflow:loss = 0.011651941, step = 22300 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.951\n",
      "INFO:tensorflow:loss = 0.021625653, step = 22400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 704.113\n",
      "INFO:tensorflow:loss = 0.03162015, step = 22500 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.458\n",
      "INFO:tensorflow:loss = 0.009711761, step = 22600 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 660.355\n",
      "INFO:tensorflow:loss = 0.021304015, step = 22700 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.279\n",
      "INFO:tensorflow:loss = 0.0015813148, step = 22800 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.98\n",
      "INFO:tensorflow:loss = 0.02138229, step = 22900 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 706.49\n",
      "INFO:tensorflow:loss = 0.024424132, step = 23000 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.911\n",
      "INFO:tensorflow:loss = 0.026397526, step = 23100 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.335\n",
      "INFO:tensorflow:loss = 0.035993647, step = 23200 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 690.03\n",
      "INFO:tensorflow:loss = 0.059697825, step = 23300 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 674.348\n",
      "INFO:tensorflow:loss = 0.0054194727, step = 23400 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 666.184\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.0015951331, step = 23500 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 716.748\n",
      "INFO:tensorflow:loss = 0.01448201, step = 23600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.357\n",
      "INFO:tensorflow:loss = 0.023917893, step = 23700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.167\n",
      "INFO:tensorflow:loss = 0.020400975, step = 23800 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 753.404\n",
      "INFO:tensorflow:loss = 0.14286822, step = 23900 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 708.009\n",
      "INFO:tensorflow:loss = 0.0020566902, step = 24000 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 643.3\n",
      "INFO:tensorflow:loss = 0.15405758, step = 24100 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.752\n",
      "INFO:tensorflow:loss = 0.015070706, step = 24200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.192\n",
      "INFO:tensorflow:loss = 0.02393239, step = 24300 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.566\n",
      "INFO:tensorflow:loss = 0.034916855, step = 24400 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.1\n",
      "INFO:tensorflow:loss = 0.013192015, step = 24500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 658.564\n",
      "INFO:tensorflow:loss = 0.17433313, step = 24600 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.9\n",
      "INFO:tensorflow:loss = 0.035164796, step = 24700 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.366\n",
      "INFO:tensorflow:loss = 0.04726214, step = 24800 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.215\n",
      "INFO:tensorflow:loss = 0.026133917, step = 24900 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.795\n",
      "INFO:tensorflow:loss = 0.012287105, step = 25000 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 690.065\n",
      "INFO:tensorflow:loss = 0.044558622, step = 25100 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.008\n",
      "INFO:tensorflow:loss = 0.022052187, step = 25200 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 793.056\n",
      "INFO:tensorflow:loss = 0.030693207, step = 25300 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.059\n",
      "INFO:tensorflow:loss = 0.009593873, step = 25400 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 689.882\n",
      "INFO:tensorflow:loss = 0.0099054035, step = 25500 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 677.624\n",
      "INFO:tensorflow:loss = 0.0378652, step = 25600 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 780.923\n",
      "INFO:tensorflow:loss = 0.05418991, step = 25700 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.105\n",
      "INFO:tensorflow:loss = 0.00080424885, step = 25800 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 804.837\n",
      "INFO:tensorflow:loss = 0.06874226, step = 25900 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 736.951\n",
      "INFO:tensorflow:loss = 0.106181085, step = 26000 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.673\n",
      "INFO:tensorflow:loss = 0.005992038, step = 26100 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.985\n",
      "INFO:tensorflow:loss = 0.035752, step = 26200 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.419\n",
      "INFO:tensorflow:loss = 0.0062805437, step = 26300 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.034\n",
      "INFO:tensorflow:loss = 0.027117174, step = 26400 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.238\n",
      "INFO:tensorflow:loss = 0.012986038, step = 26500 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 747.857\n",
      "INFO:tensorflow:loss = 0.02381338, step = 26600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.532\n",
      "INFO:tensorflow:loss = 0.016095651, step = 26700 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 668.293\n",
      "INFO:tensorflow:loss = 0.02392416, step = 26800 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 749.81\n",
      "INFO:tensorflow:loss = 0.03930056, step = 26900 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 715.178\n",
      "INFO:tensorflow:loss = 0.040683754, step = 27000 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.319\n",
      "INFO:tensorflow:loss = 0.078643784, step = 27100 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 729.727\n",
      "INFO:tensorflow:loss = 0.008082557, step = 27200 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 695.678\n",
      "INFO:tensorflow:loss = 0.033514295, step = 27300 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 754.181\n",
      "INFO:tensorflow:loss = 0.027759574, step = 27400 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 661.733\n",
      "INFO:tensorflow:loss = 0.05964483, step = 27500 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 721.166\n",
      "INFO:tensorflow:loss = 0.01151644, step = 27600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 666.503\n",
      "INFO:tensorflow:loss = 0.015075784, step = 27700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 690.419\n",
      "INFO:tensorflow:loss = 0.018286407, step = 27800 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 693.63\n",
      "INFO:tensorflow:loss = 0.06667967, step = 27900 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 671.273\n",
      "INFO:tensorflow:loss = 0.014374517, step = 28000 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 709.502\n",
      "INFO:tensorflow:loss = 0.10888465, step = 28100 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 796.051\n",
      "INFO:tensorflow:loss = 0.008978837, step = 28200 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 719.562\n",
      "INFO:tensorflow:loss = 0.03003839, step = 28300 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 646.783\n",
      "INFO:tensorflow:loss = 0.0578579, step = 28400 (0.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 661.338\n",
      "INFO:tensorflow:loss = 0.011548916, step = 28500 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.989\n",
      "INFO:tensorflow:loss = 0.06817261, step = 28600 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 703.936\n",
      "INFO:tensorflow:loss = 0.02043501, step = 28700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.376\n",
      "INFO:tensorflow:loss = 0.032203794, step = 28800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 767.124\n",
      "INFO:tensorflow:loss = 0.016585821, step = 28900 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 742.731\n",
      "INFO:tensorflow:loss = 0.04387374, step = 29000 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 639.467\n",
      "INFO:tensorflow:loss = 0.0045471583, step = 29100 (0.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 740.898\n",
      "INFO:tensorflow:loss = 0.0015986088, step = 29200 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.329\n",
      "INFO:tensorflow:loss = 0.03186493, step = 29300 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.269\n",
      "INFO:tensorflow:loss = 0.032493334, step = 29400 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.522\n",
      "INFO:tensorflow:loss = 0.0047063627, step = 29500 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.874\n",
      "INFO:tensorflow:loss = 0.0099709965, step = 29600 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 717.068\n",
      "INFO:tensorflow:loss = 0.030789599, step = 29700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.432\n",
      "INFO:tensorflow:loss = 0.017368333, step = 29800 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 742.131\n",
      "INFO:tensorflow:loss = 0.015394726, step = 29900 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.964\n",
      "INFO:tensorflow:loss = 0.024769576, step = 30000 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 701.976\n",
      "INFO:tensorflow:loss = 0.01302583, step = 30100 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 797.558\n",
      "INFO:tensorflow:loss = 0.006958114, step = 30200 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 651.51\n",
      "INFO:tensorflow:loss = 0.006674879, step = 30300 (0.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 768.814\n",
      "INFO:tensorflow:loss = 0.022546686, step = 30400 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.478\n",
      "INFO:tensorflow:loss = 0.0409905, step = 30500 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.33\n",
      "INFO:tensorflow:loss = 0.0010423085, step = 30600 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 770.57\n",
      "INFO:tensorflow:loss = 0.016143773, step = 30700 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 685.84\n",
      "INFO:tensorflow:loss = 0.005080305, step = 30800 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 653.954\n",
      "INFO:tensorflow:loss = 0.0001563972, step = 30900 (0.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 639.838\n",
      "INFO:tensorflow:loss = 0.016453827, step = 31000 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 687.275\n",
      "INFO:tensorflow:loss = 0.018924043, step = 31100 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 675.629\n",
      "INFO:tensorflow:loss = 0.033844046, step = 31200 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 753.373\n",
      "INFO:tensorflow:loss = 0.02877907, step = 31300 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.878\n",
      "INFO:tensorflow:loss = 0.0059439233, step = 31400 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 656.713\n",
      "INFO:tensorflow:loss = 0.009549853, step = 31500 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 720.365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.02079692, step = 31600 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 832.499\n",
      "INFO:tensorflow:loss = 0.0025147125, step = 31700 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.332\n",
      "INFO:tensorflow:loss = 0.007999756, step = 31800 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.276\n",
      "INFO:tensorflow:loss = 0.01664688, step = 31900 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 704.34\n",
      "INFO:tensorflow:loss = 0.019514482, step = 32000 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.025\n",
      "INFO:tensorflow:loss = 0.015806645, step = 32100 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.173\n",
      "INFO:tensorflow:loss = 0.019019779, step = 32200 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 760.044\n",
      "INFO:tensorflow:loss = 0.032663573, step = 32300 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 703.202\n",
      "INFO:tensorflow:loss = 0.011111577, step = 32400 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.917\n",
      "INFO:tensorflow:loss = 0.0067031216, step = 32500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.655\n",
      "INFO:tensorflow:loss = 0.065009676, step = 32600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.491\n",
      "INFO:tensorflow:loss = 0.0190825, step = 32700 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.206\n",
      "INFO:tensorflow:loss = 0.042236935, step = 32800 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.888\n",
      "INFO:tensorflow:loss = 0.019598864, step = 32900 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 691.011\n",
      "INFO:tensorflow:loss = 0.037666045, step = 33000 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 707.653\n",
      "INFO:tensorflow:loss = 0.00819896, step = 33100 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.321\n",
      "INFO:tensorflow:loss = 0.008802559, step = 33200 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 739.991\n",
      "INFO:tensorflow:loss = 0.0069321506, step = 33300 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 793.851\n",
      "INFO:tensorflow:loss = 0.011861758, step = 33400 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 692.6\n",
      "INFO:tensorflow:loss = 0.005425592, step = 33500 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 689.817\n",
      "INFO:tensorflow:loss = 0.06730152, step = 33600 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 684.068\n",
      "INFO:tensorflow:loss = 0.009559411, step = 33700 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.27\n",
      "INFO:tensorflow:loss = 0.011085326, step = 33800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 684.8\n",
      "INFO:tensorflow:loss = 0.0030444951, step = 33900 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.327\n",
      "INFO:tensorflow:loss = 0.03442745, step = 34000 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.378\n",
      "INFO:tensorflow:loss = 0.019509332, step = 34100 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 690.905\n",
      "INFO:tensorflow:loss = 0.017489627, step = 34200 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 692.558\n",
      "INFO:tensorflow:loss = 0.01938666, step = 34300 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.012\n",
      "INFO:tensorflow:loss = 0.01571033, step = 34400 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 695.644\n",
      "INFO:tensorflow:loss = 0.05608541, step = 34500 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 729.396\n",
      "INFO:tensorflow:loss = 0.0026731805, step = 34600 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 703.836\n",
      "INFO:tensorflow:loss = 0.05696653, step = 34700 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 660.879\n",
      "INFO:tensorflow:loss = 0.004377494, step = 34800 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.866\n",
      "INFO:tensorflow:loss = 0.0021145523, step = 34900 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 789.812\n",
      "INFO:tensorflow:loss = 0.01441968, step = 35000 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.295\n",
      "INFO:tensorflow:loss = 0.0029570013, step = 35100 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.584\n",
      "INFO:tensorflow:loss = 0.009255512, step = 35200 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 723.079\n",
      "INFO:tensorflow:loss = 0.03133426, step = 35300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.35\n",
      "INFO:tensorflow:loss = 0.047382377, step = 35400 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.098\n",
      "INFO:tensorflow:loss = 0.005345381, step = 35500 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 667.808\n",
      "INFO:tensorflow:loss = 0.015965488, step = 35600 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 659.715\n",
      "INFO:tensorflow:loss = 0.000686765, step = 35700 (0.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.421\n",
      "INFO:tensorflow:loss = 0.017110772, step = 35800 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 802.368\n",
      "INFO:tensorflow:loss = 0.0051504895, step = 35900 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 808.7\n",
      "INFO:tensorflow:loss = 0.004764241, step = 36000 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.415\n",
      "INFO:tensorflow:loss = 0.002746779, step = 36100 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 689.876\n",
      "INFO:tensorflow:loss = 0.028700832, step = 36200 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 694.366\n",
      "INFO:tensorflow:loss = 0.0064910976, step = 36300 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 661.667\n",
      "INFO:tensorflow:loss = 0.010819282, step = 36400 (0.151 sec)\n",
      "INFO:tensorflow:global_step/sec: 806.527\n",
      "INFO:tensorflow:loss = 0.022298945, step = 36500 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 683.156\n",
      "INFO:tensorflow:loss = 0.0021979343, step = 36600 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 733.352\n",
      "INFO:tensorflow:loss = 0.025101447, step = 36700 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.074\n",
      "INFO:tensorflow:loss = 0.00788566, step = 36800 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.766\n",
      "INFO:tensorflow:loss = 0.0019166737, step = 36900 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.892\n",
      "INFO:tensorflow:loss = 0.012619713, step = 37000 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.681\n",
      "INFO:tensorflow:loss = 0.013296781, step = 37100 (0.155 sec)\n",
      "INFO:tensorflow:global_step/sec: 697.868\n",
      "INFO:tensorflow:loss = 0.024330642, step = 37200 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.778\n",
      "INFO:tensorflow:loss = 0.014205432, step = 37300 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 738.04\n",
      "INFO:tensorflow:loss = 0.018223232, step = 37400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.235\n",
      "INFO:tensorflow:loss = 0.024151746, step = 37500 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.103\n",
      "INFO:tensorflow:loss = 0.01054293, step = 37600 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 833.851\n",
      "INFO:tensorflow:loss = 0.015046421, step = 37700 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 705.7\n",
      "INFO:tensorflow:loss = 0.0017428349, step = 37800 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 731.222\n",
      "INFO:tensorflow:loss = 0.005485077, step = 37900 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 741.912\n",
      "INFO:tensorflow:loss = 0.03764577, step = 38000 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 734.208\n",
      "INFO:tensorflow:loss = 0.033232145, step = 38100 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.329\n",
      "INFO:tensorflow:loss = 0.0037946901, step = 38200 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 672.011\n",
      "INFO:tensorflow:loss = 0.0031624357, step = 38300 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 678.642\n",
      "INFO:tensorflow:loss = 0.03833987, step = 38400 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 729.256\n",
      "INFO:tensorflow:loss = 0.022040468, step = 38500 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 710.484\n",
      "INFO:tensorflow:loss = 0.021502703, step = 38600 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.009\n",
      "INFO:tensorflow:loss = 0.017402876, step = 38700 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 698.552\n",
      "INFO:tensorflow:loss = 0.015710965, step = 38800 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.205\n",
      "INFO:tensorflow:loss = 0.042211603, step = 38900 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.404\n",
      "INFO:tensorflow:loss = 0.012911957, step = 39000 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 692.128\n",
      "INFO:tensorflow:loss = 0.016184535, step = 39100 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 763.318\n",
      "INFO:tensorflow:loss = 0.015847027, step = 39200 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 667.015\n",
      "INFO:tensorflow:loss = 0.021537956, step = 39300 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 690.515\n",
      "INFO:tensorflow:loss = 0.0031349366, step = 39400 (0.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.931\n",
      "INFO:tensorflow:loss = 0.0049741194, step = 39500 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 755.075\n",
      "INFO:tensorflow:loss = 0.02600574, step = 39600 (0.132 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 710.589\n",
      "INFO:tensorflow:loss = 0.006068499, step = 39700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.696\n",
      "INFO:tensorflow:loss = 0.017294474, step = 39800 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 699.222\n",
      "INFO:tensorflow:loss = 0.012692339, step = 39900 (0.143 sec)\n",
      "INFO:tensorflow:global_step/sec: 752.594\n",
      "INFO:tensorflow:loss = 0.04729276, step = 40000 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.097\n",
      "INFO:tensorflow:loss = 0.008243736, step = 40100 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.43\n",
      "INFO:tensorflow:loss = 0.00960378, step = 40200 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 714.436\n",
      "INFO:tensorflow:loss = 0.035596434, step = 40300 (0.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.992\n",
      "INFO:tensorflow:loss = 0.057535127, step = 40400 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.32\n",
      "INFO:tensorflow:loss = 0.0015139935, step = 40500 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.001\n",
      "INFO:tensorflow:loss = 0.0012848399, step = 40600 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 636.647\n",
      "INFO:tensorflow:loss = 0.009264048, step = 40700 (0.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 697.209\n",
      "INFO:tensorflow:loss = 0.024688, step = 40800 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 712.774\n",
      "INFO:tensorflow:loss = 0.0015447914, step = 40900 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 751.83\n",
      "INFO:tensorflow:loss = 0.019551061, step = 41000 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.354\n",
      "INFO:tensorflow:loss = 0.010595959, step = 41100 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.937\n",
      "INFO:tensorflow:loss = 0.0028679944, step = 41200 (0.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 737.975\n",
      "INFO:tensorflow:loss = 0.022258334, step = 41300 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 696.261\n",
      "INFO:tensorflow:loss = 0.00084078737, step = 41400 (0.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 711.678\n",
      "INFO:tensorflow:loss = 0.07653401, step = 41500 (0.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 706.256\n",
      "INFO:tensorflow:loss = 0.0024864883, step = 41600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 669.921\n",
      "INFO:tensorflow:loss = 0.045245055, step = 41700 (0.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.434\n",
      "INFO:tensorflow:loss = 0.023051996, step = 41800 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 730.237\n",
      "INFO:tensorflow:loss = 0.000904014, step = 41900 (0.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 750.778\n",
      "INFO:tensorflow:loss = 0.0008925211, step = 42000 (0.133 sec)\n",
      "INFO:tensorflow:global_step/sec: 703.198\n",
      "INFO:tensorflow:loss = 0.02051543, step = 42100 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 679.881\n",
      "INFO:tensorflow:loss = 0.014940187, step = 42200 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 686.019\n",
      "INFO:tensorflow:loss = 0.014624184, step = 42300 (0.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.501\n",
      "INFO:tensorflow:loss = 0.0051962715, step = 42400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 664.64\n",
      "INFO:tensorflow:loss = 0.012029415, step = 42500 (0.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 758.243\n",
      "INFO:tensorflow:loss = 0.005246341, step = 42600 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 779.919\n",
      "INFO:tensorflow:loss = 0.00075935933, step = 42700 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 744.733\n",
      "INFO:tensorflow:loss = 0.008091343, step = 42800 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 735.747\n",
      "INFO:tensorflow:loss = 0.0122832265, step = 42900 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 713.299\n",
      "INFO:tensorflow:loss = 0.018309314, step = 43000 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 700.501\n",
      "INFO:tensorflow:loss = 0.014963597, step = 43100 (0.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 785.6\n",
      "INFO:tensorflow:loss = 0.01887432, step = 43200 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 725.233\n",
      "INFO:tensorflow:loss = 0.010414865, step = 43300 (0.138 sec)\n",
      "INFO:tensorflow:global_step/sec: 748.593\n",
      "INFO:tensorflow:loss = 0.015046911, step = 43400 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 732.268\n",
      "INFO:tensorflow:loss = 0.0080095865, step = 43500 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 680.628\n",
      "INFO:tensorflow:loss = 0.008756702, step = 43600 (0.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 724.501\n",
      "INFO:tensorflow:loss = 0.007790994, step = 43700 (0.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 752.679\n",
      "INFO:tensorflow:loss = 0.008473238, step = 43800 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 745.042\n",
      "INFO:tensorflow:loss = 0.005319404, step = 43900 (0.135 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 44000 into /tmp/tmpenr8wre6/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.019557934.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.estimator.canned.dnn.DNNClassifier at 0x7fadb0686898>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols = [tf.feature_column.numeric_column(\"X\", shape=[28 * 28])]\n",
    "dnn_clf = tf.estimator.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                     feature_columns=feature_cols)\n",
    "\n",
    "input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_train}, y=y_train, num_epochs=40, batch_size=50, shuffle=True)\n",
    "dnn_clf.train(input_fn=input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-23-04:16:26\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpenr8wre6/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-23-04:16:26\n",
      "INFO:tensorflow:Saving dict for global step 44000: accuracy = 0.9807, average_loss = 0.10106682, global_step = 44000, loss = 12.793268\n"
     ]
    }
   ],
   "source": [
    "test_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"X\": X_test}, y=y_test, shuffle=False)\n",
    "eval_results = dnn_clf.evaluate(input_fn=test_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9807,\n",
       " 'average_loss': 0.10106682,\n",
       " 'loss': 12.793268,\n",
       " 'global_step': 44000}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpenr8wre6/model.ckpt-44000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'logits': array([ -8.736278 ,  -4.2979555,  -1.4217235,   5.8501   ,  -7.116479 ,\n",
       "         -6.524613 , -15.091009 ,  24.78048  ,  -3.6320136,   1.4088718],\n",
       "       dtype=float32),\n",
       " 'probabilities': array([2.77879767e-15, 2.35177925e-13, 4.17376030e-12, 6.00676264e-09,\n",
       "        1.40387164e-14, 2.53729736e-14, 4.83094033e-18, 1.00000000e+00,\n",
       "        4.57731943e-13, 7.07685160e-11], dtype=float32),\n",
       " 'class_ids': array([7]),\n",
       " 'classes': array([b'7'], dtype=object)}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_iter = dnn_clf.predict(input_fn=test_input_fn)\n",
    "y_pred = list(y_pred_iter)\n",
    "y_pred[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `tf.contrib.learn`을 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# from tensorflow.examples.tutorials.mnist import input_data\n",
    "# mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = mnist.train.images\n",
    "# X_test = mnist.test.images\n",
    "# y_train = mnist.train.labels.astype(\"int\")\n",
    "# y_test = mnist.test.labels.astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-18-0efa993935b9>:1: RunConfig.__init__ (from tensorflow.contrib.learn.python.learn.estimators.run_config) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.RunConfig instead.\n",
      "WARNING:tensorflow:From <ipython-input-18-0efa993935b9>:3: infer_real_valued_columns_from_input (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please specify feature columns explicitly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:142: setup_train_data_feeder (from tensorflow.contrib.learn.python.learn.learn_io.data_feeder) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:100: extract_pandas_data (from tensorflow.contrib.learn.python.learn.learn_io.pandas_io) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please access pandas data directly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:159: DataFeeder.__init__ (from tensorflow.contrib.learn.python.learn.learn_io.data_feeder) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:340: check_array (from tensorflow.contrib.learn.python.learn.learn_io.data_feeder) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please convert numpy dtypes explicitly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:182: infer_real_valued_columns_from_input_fn (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please specify feature columns explicitly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/dnn.py:378: multi_class_head (from tensorflow.contrib.learn.python.learn.estimators.head) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.contrib.estimator.*_head.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py:1179: BaseEstimator.__init__ (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please replace uses of any Estimator from tf.contrib.learn with an Estimator from tf.estimator.*\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpvld1kthq\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fad68704080>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': 42, '_save_summary_steps': 100, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_session_config': None, '_save_checkpoints_steps': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': '/tmp/tmpvld1kthq'}\n",
      "WARNING:tensorflow:From <ipython-input-18-0efa993935b9>:6: SKCompat.__init__ (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to the Estimator interface.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py:102: extract_pandas_labels (from tensorflow.contrib.learn.python.learn.learn_io.pandas_io) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please access pandas data directly.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/estimators/head.py:678: ModelFnOps.__new__ (from tensorflow.contrib.learn.python.learn.estimators.model_fn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.EstimatorSpec. You can use the `estimator_spec` method to create an equivalent one.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpvld1kthq/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.4005778, step = 0\n",
      "INFO:tensorflow:global_step/sec: 725.415\n",
      "INFO:tensorflow:loss = 0.31267592, step = 100 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 785.736\n",
      "INFO:tensorflow:loss = 0.29636884, step = 200 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 907.177\n",
      "INFO:tensorflow:loss = 0.4080767, step = 300 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 833.98\n",
      "INFO:tensorflow:loss = 0.23435771, step = 400 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 873.177\n",
      "INFO:tensorflow:loss = 0.2913078, step = 500 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 885.681\n",
      "INFO:tensorflow:loss = 0.07167665, step = 600 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 725.751\n",
      "INFO:tensorflow:loss = 0.14161882, step = 700 (0.139 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.056\n",
      "INFO:tensorflow:loss = 0.20138773, step = 800 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 839.364\n",
      "INFO:tensorflow:loss = 0.11518536, step = 900 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 757.947\n",
      "INFO:tensorflow:loss = 0.24996708, step = 1000 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 762.992\n",
      "INFO:tensorflow:loss = 0.18783642, step = 1100 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 864.219\n",
      "INFO:tensorflow:loss = 0.1528608, step = 1200 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 886.307\n",
      "INFO:tensorflow:loss = 0.18098918, step = 1300 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 905.027\n",
      "INFO:tensorflow:loss = 0.075340584, step = 1400 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 795.983\n",
      "INFO:tensorflow:loss = 0.120701864, step = 1500 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 805.492\n",
      "INFO:tensorflow:loss = 0.08812464, step = 1600 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 881.556\n",
      "INFO:tensorflow:loss = 0.038204927, step = 1700 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.757\n",
      "INFO:tensorflow:loss = 0.15442611, step = 1800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 790.415\n",
      "INFO:tensorflow:loss = 0.10359758, step = 1900 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 885.708\n",
      "INFO:tensorflow:loss = 0.11606234, step = 2000 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 927.332\n",
      "INFO:tensorflow:loss = 0.019048026, step = 2100 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 877.723\n",
      "INFO:tensorflow:loss = 0.023819495, step = 2200 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.789\n",
      "INFO:tensorflow:loss = 0.06001607, step = 2300 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 904.303\n",
      "INFO:tensorflow:loss = 0.057904292, step = 2400 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.292\n",
      "INFO:tensorflow:loss = 0.09638496, step = 2500 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 832.076\n",
      "INFO:tensorflow:loss = 0.052304104, step = 2600 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 908.391\n",
      "INFO:tensorflow:loss = 0.016296867, step = 2700 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 960.848\n",
      "INFO:tensorflow:loss = 0.056792654, step = 2800 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 910.892\n",
      "INFO:tensorflow:loss = 0.17416798, step = 2900 (0.110 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 815.468\n",
      "INFO:tensorflow:loss = 0.01614511, step = 3000 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 965.495\n",
      "INFO:tensorflow:loss = 0.046176963, step = 3100 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 956.549\n",
      "INFO:tensorflow:loss = 0.009885453, step = 3200 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 868.109\n",
      "INFO:tensorflow:loss = 0.039482597, step = 3300 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 925.235\n",
      "INFO:tensorflow:loss = 0.25074363, step = 3400 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 881.903\n",
      "INFO:tensorflow:loss = 0.108091794, step = 3500 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 835.948\n",
      "INFO:tensorflow:loss = 0.19296156, step = 3600 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 911.595\n",
      "INFO:tensorflow:loss = 0.032442845, step = 3700 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 933.987\n",
      "INFO:tensorflow:loss = 0.009454474, step = 3800 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 827.347\n",
      "INFO:tensorflow:loss = 0.049338464, step = 3900 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 905.659\n",
      "INFO:tensorflow:loss = 0.11530109, step = 4000 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 949.305\n",
      "INFO:tensorflow:loss = 0.028121071, step = 4100 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 972.865\n",
      "INFO:tensorflow:loss = 0.058370374, step = 4200 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 878.05\n",
      "INFO:tensorflow:loss = 0.17256807, step = 4300 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 879.479\n",
      "INFO:tensorflow:loss = 0.14077905, step = 4400 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 934.981\n",
      "INFO:tensorflow:loss = 0.01495807, step = 4500 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 858.611\n",
      "INFO:tensorflow:loss = 0.015784886, step = 4600 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 913.885\n",
      "INFO:tensorflow:loss = 0.011284053, step = 4700 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 910.118\n",
      "INFO:tensorflow:loss = 0.02258128, step = 4800 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 776.138\n",
      "INFO:tensorflow:loss = 0.09316278, step = 4900 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 852.948\n",
      "INFO:tensorflow:loss = 0.091467656, step = 5000 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 864.114\n",
      "INFO:tensorflow:loss = 0.013380401, step = 5100 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 897.024\n",
      "INFO:tensorflow:loss = 0.02873302, step = 5200 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 929.587\n",
      "INFO:tensorflow:loss = 0.032953404, step = 5300 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 882.599\n",
      "INFO:tensorflow:loss = 0.026337178, step = 5400 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 852.882\n",
      "INFO:tensorflow:loss = 0.018157216, step = 5500 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 894.142\n",
      "INFO:tensorflow:loss = 0.040303618, step = 5600 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 835.424\n",
      "INFO:tensorflow:loss = 0.009256579, step = 5700 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 902.348\n",
      "INFO:tensorflow:loss = 0.006459822, step = 5800 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 867.181\n",
      "INFO:tensorflow:loss = 0.066927515, step = 5900 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 878.898\n",
      "INFO:tensorflow:loss = 0.082413, step = 6000 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 862.731\n",
      "INFO:tensorflow:loss = 0.016897984, step = 6100 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 776.663\n",
      "INFO:tensorflow:loss = 0.015233619, step = 6200 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 1038.86\n",
      "INFO:tensorflow:loss = 0.046267062, step = 6300 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 884.499\n",
      "INFO:tensorflow:loss = 0.036248315, step = 6400 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 766.523\n",
      "INFO:tensorflow:loss = 0.0092446245, step = 6500 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 948.257\n",
      "INFO:tensorflow:loss = 0.018753435, step = 6600 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 916.758\n",
      "INFO:tensorflow:loss = 0.014870468, step = 6700 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 908.255\n",
      "INFO:tensorflow:loss = 0.005744975, step = 6800 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 837.223\n",
      "INFO:tensorflow:loss = 0.012360539, step = 6900 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 939.258\n",
      "INFO:tensorflow:loss = 0.019316781, step = 7000 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 888.793\n",
      "INFO:tensorflow:loss = 0.006693481, step = 7100 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 843.32\n",
      "INFO:tensorflow:loss = 0.06590567, step = 7200 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 924.246\n",
      "INFO:tensorflow:loss = 0.0091822855, step = 7300 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 880.435\n",
      "INFO:tensorflow:loss = 0.02041543, step = 7400 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 840.752\n",
      "INFO:tensorflow:loss = 0.00923355, step = 7500 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 973.617\n",
      "INFO:tensorflow:loss = 0.012516694, step = 7600 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 792.673\n",
      "INFO:tensorflow:loss = 0.006828384, step = 7700 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 830.795\n",
      "INFO:tensorflow:loss = 0.005702033, step = 7800 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 845.923\n",
      "INFO:tensorflow:loss = 0.0060084, step = 7900 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 825.033\n",
      "INFO:tensorflow:loss = 0.0028837053, step = 8000 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 988.691\n",
      "INFO:tensorflow:loss = 0.0065821256, step = 8100 (0.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 889.987\n",
      "INFO:tensorflow:loss = 0.038388062, step = 8200 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 985.93\n",
      "INFO:tensorflow:loss = 0.028459603, step = 8300 (0.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 736.128\n",
      "INFO:tensorflow:loss = 0.006259576, step = 8400 (0.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 897.517\n",
      "INFO:tensorflow:loss = 0.0049914243, step = 8500 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 929.218\n",
      "INFO:tensorflow:loss = 0.0058809943, step = 8600 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 867.883\n",
      "INFO:tensorflow:loss = 0.0058761914, step = 8700 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 873.983\n",
      "INFO:tensorflow:loss = 0.0063133854, step = 8800 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 862.557\n",
      "INFO:tensorflow:loss = 0.002403117, step = 8900 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 820.153\n",
      "INFO:tensorflow:loss = 0.0043475544, step = 9000 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 904.313\n",
      "INFO:tensorflow:loss = 0.009287273, step = 9100 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 943.282\n",
      "INFO:tensorflow:loss = 0.007235099, step = 9200 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 867.059\n",
      "INFO:tensorflow:loss = 0.00553244, step = 9300 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 856.428\n",
      "INFO:tensorflow:loss = 0.06505666, step = 9400 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 892.785\n",
      "INFO:tensorflow:loss = 0.0031202447, step = 9500 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.379\n",
      "INFO:tensorflow:loss = 0.01915066, step = 9600 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 910.096\n",
      "INFO:tensorflow:loss = 0.0072076977, step = 9700 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 792.576\n",
      "INFO:tensorflow:loss = 0.0015239312, step = 9800 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 987.574\n",
      "INFO:tensorflow:loss = 0.014900117, step = 9900 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 931.838\n",
      "INFO:tensorflow:loss = 0.006614263, step = 10000 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 876.045\n",
      "INFO:tensorflow:loss = 0.004810418, step = 10100 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 871.048\n",
      "INFO:tensorflow:loss = 0.011792469, step = 10200 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 800.041\n",
      "INFO:tensorflow:loss = 0.0020595554, step = 10300 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 825.966\n",
      "INFO:tensorflow:loss = 0.0062876553, step = 10400 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 915.437\n",
      "INFO:tensorflow:loss = 0.003435815, step = 10500 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 972.074\n",
      "INFO:tensorflow:loss = 0.017278777, step = 10600 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 842.742\n",
      "INFO:tensorflow:loss = 0.031814165, step = 10700 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 803.314\n",
      "INFO:tensorflow:loss = 0.0068269507, step = 10800 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 986.027\n",
      "INFO:tensorflow:loss = 0.0013256436, step = 10900 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 796.457\n",
      "INFO:tensorflow:loss = 0.01892974, step = 11000 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 989.538\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.0038287914, step = 11100 (0.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 806.092\n",
      "INFO:tensorflow:loss = 0.0010031462, step = 11200 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 950.943\n",
      "INFO:tensorflow:loss = 0.006125767, step = 11300 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 845.019\n",
      "INFO:tensorflow:loss = 0.0052804668, step = 11401 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 925.541\n",
      "INFO:tensorflow:loss = 0.012757082, step = 11500 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 898.498\n",
      "INFO:tensorflow:loss = 0.0011948881, step = 11600 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 913.056\n",
      "INFO:tensorflow:loss = 0.002479272, step = 11700 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 943.896\n",
      "INFO:tensorflow:loss = 0.0005506238, step = 11800 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 842.949\n",
      "INFO:tensorflow:loss = 0.006492114, step = 11900 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 876.703\n",
      "INFO:tensorflow:loss = 0.0005815428, step = 12000 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 913.42\n",
      "INFO:tensorflow:loss = 0.0030103605, step = 12100 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 846.484\n",
      "INFO:tensorflow:loss = 0.002930471, step = 12200 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 870.108\n",
      "INFO:tensorflow:loss = 0.005122689, step = 12300 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 901.04\n",
      "INFO:tensorflow:loss = 0.00041177266, step = 12400 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 869.756\n",
      "INFO:tensorflow:loss = 0.001255448, step = 12500 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 862.13\n",
      "INFO:tensorflow:loss = 0.0027376963, step = 12600 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 959.784\n",
      "INFO:tensorflow:loss = 0.003147854, step = 12700 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 934.006\n",
      "INFO:tensorflow:loss = 0.012359958, step = 12800 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 866.671\n",
      "INFO:tensorflow:loss = 0.0025297587, step = 12900 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 846.563\n",
      "INFO:tensorflow:loss = 0.005472062, step = 13000 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 932.54\n",
      "INFO:tensorflow:loss = 0.0029778332, step = 13100 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 924.031\n",
      "INFO:tensorflow:loss = 0.00165595, step = 13200 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 915.283\n",
      "INFO:tensorflow:loss = 0.009083047, step = 13300 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 962.756\n",
      "INFO:tensorflow:loss = 0.0071698856, step = 13400 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 904.469\n",
      "INFO:tensorflow:loss = 0.004175674, step = 13500 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 938.283\n",
      "INFO:tensorflow:loss = 0.0038817683, step = 13600 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 958.781\n",
      "INFO:tensorflow:loss = 0.0019649141, step = 13700 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 861.588\n",
      "INFO:tensorflow:loss = 0.005527034, step = 13800 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 851.516\n",
      "INFO:tensorflow:loss = 0.0036680114, step = 13900 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 865.267\n",
      "INFO:tensorflow:loss = 0.0026358152, step = 14000 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 896.502\n",
      "INFO:tensorflow:loss = 0.009617847, step = 14100 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 861.421\n",
      "INFO:tensorflow:loss = 0.0047839712, step = 14200 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 877.06\n",
      "INFO:tensorflow:loss = 0.00091536687, step = 14300 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 885.252\n",
      "INFO:tensorflow:loss = 0.001287969, step = 14400 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 879.455\n",
      "INFO:tensorflow:loss = 0.0007403266, step = 14500 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 793.735\n",
      "INFO:tensorflow:loss = 0.0044146352, step = 14600 (0.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 827.13\n",
      "INFO:tensorflow:loss = 0.0010798331, step = 14700 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 780.325\n",
      "INFO:tensorflow:loss = 0.0012260921, step = 14800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 898.671\n",
      "INFO:tensorflow:loss = 0.0023467324, step = 14900 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 911.942\n",
      "INFO:tensorflow:loss = 0.0013190353, step = 15001 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 933.381\n",
      "INFO:tensorflow:loss = 0.0021132922, step = 15100 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 972.554\n",
      "INFO:tensorflow:loss = 0.0011075395, step = 15200 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 902.64\n",
      "INFO:tensorflow:loss = 0.0021117656, step = 15300 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 850.135\n",
      "INFO:tensorflow:loss = 0.0029424548, step = 15400 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 1024.12\n",
      "INFO:tensorflow:loss = 0.005543447, step = 15500 (0.098 sec)\n",
      "INFO:tensorflow:global_step/sec: 953.283\n",
      "INFO:tensorflow:loss = 0.0029127833, step = 15600 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 902.375\n",
      "INFO:tensorflow:loss = 0.006514271, step = 15700 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 926.418\n",
      "INFO:tensorflow:loss = 0.0011431694, step = 15800 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 864.958\n",
      "INFO:tensorflow:loss = 0.0008100872, step = 15900 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 945.127\n",
      "INFO:tensorflow:loss = 0.0068712546, step = 16000 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 780.934\n",
      "INFO:tensorflow:loss = 0.0034159462, step = 16101 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 844.793\n",
      "INFO:tensorflow:loss = 0.00019279998, step = 16200 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 954.427\n",
      "INFO:tensorflow:loss = 0.0019248038, step = 16300 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 869.688\n",
      "INFO:tensorflow:loss = 0.001534009, step = 16400 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 844.325\n",
      "INFO:tensorflow:loss = 0.0018085908, step = 16500 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 884.346\n",
      "INFO:tensorflow:loss = 0.0028658912, step = 16600 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 944.569\n",
      "INFO:tensorflow:loss = 0.001987661, step = 16700 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 862.587\n",
      "INFO:tensorflow:loss = 0.0025418568, step = 16800 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 889.498\n",
      "INFO:tensorflow:loss = 0.0014058298, step = 16900 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 914.756\n",
      "INFO:tensorflow:loss = 0.003754899, step = 17000 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 826.912\n",
      "INFO:tensorflow:loss = 0.0026304917, step = 17100 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 867.038\n",
      "INFO:tensorflow:loss = 0.0020387606, step = 17200 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 983.339\n",
      "INFO:tensorflow:loss = 0.0011545033, step = 17300 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 847.636\n",
      "INFO:tensorflow:loss = 0.0015967692, step = 17400 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 821.958\n",
      "INFO:tensorflow:loss = 0.0008795982, step = 17500 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 842.629\n",
      "INFO:tensorflow:loss = 0.00068491674, step = 17600 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 885.915\n",
      "INFO:tensorflow:loss = 0.00087740173, step = 17700 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 856.671\n",
      "INFO:tensorflow:loss = 0.00017951707, step = 17800 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 846.264\n",
      "INFO:tensorflow:loss = 0.0016931241, step = 17900 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 823.865\n",
      "INFO:tensorflow:loss = 0.0005041162, step = 18000 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 916.393\n",
      "INFO:tensorflow:loss = 0.0006777653, step = 18100 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 856.582\n",
      "INFO:tensorflow:loss = 0.0031738444, step = 18200 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 888.23\n",
      "INFO:tensorflow:loss = 0.005538681, step = 18300 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 867.255\n",
      "INFO:tensorflow:loss = 0.0034323633, step = 18400 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 914.551\n",
      "INFO:tensorflow:loss = 0.007520867, step = 18500 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 924.428\n",
      "INFO:tensorflow:loss = 0.0038130968, step = 18600 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 909.654\n",
      "INFO:tensorflow:loss = 0.0018715843, step = 18700 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 947.915\n",
      "INFO:tensorflow:loss = 0.0023696977, step = 18800 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 989.935\n",
      "INFO:tensorflow:loss = 0.0023988995, step = 18900 (0.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 831.11\n",
      "INFO:tensorflow:loss = 0.0014455253, step = 19000 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 917.365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.0006202307, step = 19100 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 962.398\n",
      "INFO:tensorflow:loss = 0.0010531486, step = 19200 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 915.094\n",
      "INFO:tensorflow:loss = 0.005733006, step = 19300 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 835.875\n",
      "INFO:tensorflow:loss = 0.0018548731, step = 19400 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 890.809\n",
      "INFO:tensorflow:loss = 0.0021525905, step = 19500 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 868.618\n",
      "INFO:tensorflow:loss = 0.00076662796, step = 19600 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 911.968\n",
      "INFO:tensorflow:loss = 0.000121860365, step = 19700 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 864.429\n",
      "INFO:tensorflow:loss = 0.00048263848, step = 19800 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 979.934\n",
      "INFO:tensorflow:loss = 0.0022690005, step = 19900 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 876.877\n",
      "INFO:tensorflow:loss = 0.0020373198, step = 20000 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 899.137\n",
      "INFO:tensorflow:loss = 0.0005471564, step = 20100 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 846.693\n",
      "INFO:tensorflow:loss = 0.0024793104, step = 20200 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 893.123\n",
      "INFO:tensorflow:loss = 0.0010385577, step = 20300 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 835.138\n",
      "INFO:tensorflow:loss = 0.00013364022, step = 20400 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 914.451\n",
      "INFO:tensorflow:loss = 0.0009953488, step = 20500 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 862.937\n",
      "INFO:tensorflow:loss = 0.000725548, step = 20601 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 863.626\n",
      "INFO:tensorflow:loss = 0.00071017945, step = 20700 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 882.636\n",
      "INFO:tensorflow:loss = 0.0011815097, step = 20800 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 935.114\n",
      "INFO:tensorflow:loss = 0.0011822787, step = 20900 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 889.032\n",
      "INFO:tensorflow:loss = 0.0028882632, step = 21000 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 882.891\n",
      "INFO:tensorflow:loss = 0.0011549948, step = 21100 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 980.117\n",
      "INFO:tensorflow:loss = 0.00538647, step = 21200 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 979.616\n",
      "INFO:tensorflow:loss = 0.00066992483, step = 21300 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 1001.5\n",
      "INFO:tensorflow:loss = 0.0019436314, step = 21400 (0.100 sec)\n",
      "INFO:tensorflow:global_step/sec: 973.375\n",
      "INFO:tensorflow:loss = 0.00046895794, step = 21500 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 950.144\n",
      "INFO:tensorflow:loss = 0.00076657237, step = 21600 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 828.078\n",
      "INFO:tensorflow:loss = 0.0009052801, step = 21700 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 944.089\n",
      "INFO:tensorflow:loss = 0.0004185175, step = 21800 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 876.719\n",
      "INFO:tensorflow:loss = 0.00048334964, step = 21900 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 881.254\n",
      "INFO:tensorflow:loss = 4.9148457e-05, step = 22000 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 972.931\n",
      "INFO:tensorflow:loss = 0.00022999822, step = 22100 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 848.666\n",
      "INFO:tensorflow:loss = 0.0012118171, step = 22200 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 951.499\n",
      "INFO:tensorflow:loss = 0.0015047338, step = 22300 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 933.89\n",
      "INFO:tensorflow:loss = 0.0018024616, step = 22400 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 893.681\n",
      "INFO:tensorflow:loss = 0.0014526531, step = 22500 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 938.999\n",
      "INFO:tensorflow:loss = 0.0024839758, step = 22600 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 832.243\n",
      "INFO:tensorflow:loss = 0.0007359253, step = 22700 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 781.275\n",
      "INFO:tensorflow:loss = 0.00094723655, step = 22800 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 824.294\n",
      "INFO:tensorflow:loss = 0.0019385265, step = 22900 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 929.671\n",
      "INFO:tensorflow:loss = 0.0004997744, step = 23000 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 831.906\n",
      "INFO:tensorflow:loss = 0.002612527, step = 23100 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 888.586\n",
      "INFO:tensorflow:loss = 0.0017559311, step = 23200 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 966.044\n",
      "INFO:tensorflow:loss = 0.0020583493, step = 23300 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 764.904\n",
      "INFO:tensorflow:loss = 0.00053217163, step = 23400 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 965.511\n",
      "INFO:tensorflow:loss = 0.00085573626, step = 23500 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 855.028\n",
      "INFO:tensorflow:loss = 0.000508438, step = 23600 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 832.99\n",
      "INFO:tensorflow:loss = 0.00011113267, step = 23700 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 827.895\n",
      "INFO:tensorflow:loss = 0.0008856167, step = 23800 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 774.984\n",
      "INFO:tensorflow:loss = 0.0019630624, step = 23900 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 843.923\n",
      "INFO:tensorflow:loss = 0.0015008722, step = 24000 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 955.673\n",
      "INFO:tensorflow:loss = 0.00068334, step = 24100 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 850.668\n",
      "INFO:tensorflow:loss = 0.0017095241, step = 24200 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 885.04\n",
      "INFO:tensorflow:loss = 0.00018107546, step = 24300 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 957.217\n",
      "INFO:tensorflow:loss = 0.0018097464, step = 24400 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 857.454\n",
      "INFO:tensorflow:loss = 0.0010801455, step = 24500 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 842.898\n",
      "INFO:tensorflow:loss = 0.0008387406, step = 24600 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 967.114\n",
      "INFO:tensorflow:loss = 0.00094700116, step = 24700 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 958.805\n",
      "INFO:tensorflow:loss = 0.0011390334, step = 24800 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 958.19\n",
      "INFO:tensorflow:loss = 0.0012361046, step = 24900 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 843.132\n",
      "INFO:tensorflow:loss = 0.0006932266, step = 25000 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 950.456\n",
      "INFO:tensorflow:loss = 0.0008087346, step = 25100 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 812.935\n",
      "INFO:tensorflow:loss = 0.0016623863, step = 25200 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 882.863\n",
      "INFO:tensorflow:loss = 0.00015647997, step = 25300 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 879.374\n",
      "INFO:tensorflow:loss = 0.00058346405, step = 25400 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 950.017\n",
      "INFO:tensorflow:loss = 0.0010149479, step = 25500 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 882.317\n",
      "INFO:tensorflow:loss = 0.0007801666, step = 25600 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 773.636\n",
      "INFO:tensorflow:loss = 0.0004552298, step = 25700 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 921.276\n",
      "INFO:tensorflow:loss = 0.00075059466, step = 25800 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 810.921\n",
      "INFO:tensorflow:loss = 0.0016520639, step = 25900 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 929.751\n",
      "INFO:tensorflow:loss = 8.085647e-05, step = 26000 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 828.554\n",
      "INFO:tensorflow:loss = 0.0007698236, step = 26100 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 774.766\n",
      "INFO:tensorflow:loss = 0.001016638, step = 26201 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 925.376\n",
      "INFO:tensorflow:loss = 0.0006421384, step = 26300 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 887.396\n",
      "INFO:tensorflow:loss = 0.0011669076, step = 26400 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 948.768\n",
      "INFO:tensorflow:loss = 0.0009838426, step = 26500 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 977.49\n",
      "INFO:tensorflow:loss = 0.00048789158, step = 26600 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 938.558\n",
      "INFO:tensorflow:loss = 3.0062318e-05, step = 26700 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 882.181\n",
      "INFO:tensorflow:loss = 0.00033104292, step = 26800 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 774.011\n",
      "INFO:tensorflow:loss = 0.0008168676, step = 26900 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 855.462\n",
      "INFO:tensorflow:loss = 0.0007701437, step = 27000 (0.117 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 902.469\n",
      "INFO:tensorflow:loss = 0.0005868166, step = 27100 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 836.07\n",
      "INFO:tensorflow:loss = 0.00047166276, step = 27200 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 890.91\n",
      "INFO:tensorflow:loss = 0.0011652331, step = 27300 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 837.837\n",
      "INFO:tensorflow:loss = 0.00047435926, step = 27400 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 920.708\n",
      "INFO:tensorflow:loss = 0.00096755894, step = 27500 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 925.452\n",
      "INFO:tensorflow:loss = 0.0012103237, step = 27600 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 823.36\n",
      "INFO:tensorflow:loss = 0.0002141384, step = 27700 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 927.537\n",
      "INFO:tensorflow:loss = 0.00016483571, step = 27800 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 982.901\n",
      "INFO:tensorflow:loss = 0.0005189508, step = 27900 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 874.891\n",
      "INFO:tensorflow:loss = 0.001280538, step = 28000 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 911.974\n",
      "INFO:tensorflow:loss = 0.0006404268, step = 28100 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 901.1\n",
      "INFO:tensorflow:loss = 0.0007371138, step = 28200 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 873.477\n",
      "INFO:tensorflow:loss = 0.0008868222, step = 28300 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 856.543\n",
      "INFO:tensorflow:loss = 0.00113068, step = 28400 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 899.409\n",
      "INFO:tensorflow:loss = 0.00015523745, step = 28500 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 846.25\n",
      "INFO:tensorflow:loss = 0.0004846629, step = 28600 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 756.629\n",
      "INFO:tensorflow:loss = 0.0012733703, step = 28700 (0.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 799.745\n",
      "INFO:tensorflow:loss = 0.0010789562, step = 28800 (0.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 886\n",
      "INFO:tensorflow:loss = 0.0004019866, step = 28900 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 828.748\n",
      "INFO:tensorflow:loss = 0.0012035872, step = 29000 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 844.992\n",
      "INFO:tensorflow:loss = 0.0011771013, step = 29100 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 897.381\n",
      "INFO:tensorflow:loss = 0.0011630511, step = 29200 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 877.235\n",
      "INFO:tensorflow:loss = 0.001477482, step = 29300 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 821.613\n",
      "INFO:tensorflow:loss = 0.0011523708, step = 29400 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 967.765\n",
      "INFO:tensorflow:loss = 0.0008018672, step = 29500 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 817.012\n",
      "INFO:tensorflow:loss = 0.00025682236, step = 29600 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 959.058\n",
      "INFO:tensorflow:loss = 0.00023619819, step = 29700 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 845.52\n",
      "INFO:tensorflow:loss = 0.00039737753, step = 29800 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 860.598\n",
      "INFO:tensorflow:loss = 0.00084936613, step = 29900 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 859.714\n",
      "INFO:tensorflow:loss = 0.00017475578, step = 30000 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 1007.13\n",
      "INFO:tensorflow:loss = 0.00062488933, step = 30100 (0.099 sec)\n",
      "INFO:tensorflow:global_step/sec: 809.246\n",
      "INFO:tensorflow:loss = 0.00017820817, step = 30200 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 857.077\n",
      "INFO:tensorflow:loss = 0.00072053133, step = 30300 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 919.529\n",
      "INFO:tensorflow:loss = 0.0015941224, step = 30400 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 948.118\n",
      "INFO:tensorflow:loss = 0.00074913667, step = 30500 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 940.767\n",
      "INFO:tensorflow:loss = 0.00089774636, step = 30600 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 809.814\n",
      "INFO:tensorflow:loss = 0.00079255796, step = 30700 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 814.499\n",
      "INFO:tensorflow:loss = 0.000965737, step = 30800 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 871.95\n",
      "INFO:tensorflow:loss = 0.0005685436, step = 30900 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 888.522\n",
      "INFO:tensorflow:loss = 0.0009217605, step = 31000 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 884.531\n",
      "INFO:tensorflow:loss = 0.0011548988, step = 31100 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 860.52\n",
      "INFO:tensorflow:loss = 0.0003708408, step = 31200 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 927.121\n",
      "INFO:tensorflow:loss = 0.00048043526, step = 31300 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 891.961\n",
      "INFO:tensorflow:loss = 0.0013769933, step = 31400 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 934.397\n",
      "INFO:tensorflow:loss = 0.00017961602, step = 31500 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 864.967\n",
      "INFO:tensorflow:loss = 0.00033673577, step = 31600 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 976.521\n",
      "INFO:tensorflow:loss = 0.0005989986, step = 31700 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 809.325\n",
      "INFO:tensorflow:loss = 0.00018081676, step = 31800 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 784.608\n",
      "INFO:tensorflow:loss = 0.0007124843, step = 31900 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 891.716\n",
      "INFO:tensorflow:loss = 0.000118802505, step = 32000 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 921.27\n",
      "INFO:tensorflow:loss = 0.00050229277, step = 32100 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 868.742\n",
      "INFO:tensorflow:loss = 0.0011416804, step = 32200 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 874.413\n",
      "INFO:tensorflow:loss = 0.00053919246, step = 32300 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 787.326\n",
      "INFO:tensorflow:loss = 0.00016630821, step = 32400 (0.127 sec)\n",
      "INFO:tensorflow:global_step/sec: 894.369\n",
      "INFO:tensorflow:loss = 0.00033608876, step = 32500 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 746.93\n",
      "INFO:tensorflow:loss = 0.00024971392, step = 32600 (0.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 979.378\n",
      "INFO:tensorflow:loss = 0.00069454504, step = 32700 (0.102 sec)\n",
      "INFO:tensorflow:global_step/sec: 787.416\n",
      "INFO:tensorflow:loss = 0.00101935, step = 32800 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 885.115\n",
      "INFO:tensorflow:loss = 0.00056119816, step = 32900 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 953.19\n",
      "INFO:tensorflow:loss = 0.00069532736, step = 33000 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 950.292\n",
      "INFO:tensorflow:loss = 0.0017849223, step = 33100 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 851.855\n",
      "INFO:tensorflow:loss = 0.0008970888, step = 33200 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 775.221\n",
      "INFO:tensorflow:loss = 0.000423045, step = 33300 (0.129 sec)\n",
      "INFO:tensorflow:global_step/sec: 820.283\n",
      "INFO:tensorflow:loss = 0.0011533803, step = 33401 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 945.289\n",
      "INFO:tensorflow:loss = 0.0003587299, step = 33500 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 889.811\n",
      "INFO:tensorflow:loss = 0.0010590546, step = 33600 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 901.108\n",
      "INFO:tensorflow:loss = 0.00092741963, step = 33700 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 944.085\n",
      "INFO:tensorflow:loss = 0.000646905, step = 33800 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 834.955\n",
      "INFO:tensorflow:loss = 0.0009302563, step = 33900 (0.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 904.633\n",
      "INFO:tensorflow:loss = 0.00076165714, step = 34000 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 973.999\n",
      "INFO:tensorflow:loss = 0.00060255686, step = 34100 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 889.347\n",
      "INFO:tensorflow:loss = 0.00097624736, step = 34200 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 847.079\n",
      "INFO:tensorflow:loss = 0.00014553774, step = 34300 (0.118 sec)\n",
      "INFO:tensorflow:global_step/sec: 886.357\n",
      "INFO:tensorflow:loss = 0.00041043758, step = 34400 (0.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 954.842\n",
      "INFO:tensorflow:loss = 0.00045626852, step = 34500 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 904.846\n",
      "INFO:tensorflow:loss = 0.00045873155, step = 34600 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 812.327\n",
      "INFO:tensorflow:loss = 0.0012399855, step = 34700 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 865.858\n",
      "INFO:tensorflow:loss = 0.0006013205, step = 34800 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 777.982\n",
      "INFO:tensorflow:loss = 0.00081671565, step = 34900 (0.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 834.194\n",
      "INFO:tensorflow:loss = 0.00028821078, step = 35000 (0.120 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 925.303\n",
      "INFO:tensorflow:loss = 0.0007629123, step = 35100 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 860.988\n",
      "INFO:tensorflow:loss = 0.0002040974, step = 35200 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 878.004\n",
      "INFO:tensorflow:loss = 0.00013510644, step = 35300 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 813.302\n",
      "INFO:tensorflow:loss = 0.0012406149, step = 35400 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 1010.59\n",
      "INFO:tensorflow:loss = 0.00031413732, step = 35500 (0.099 sec)\n",
      "INFO:tensorflow:global_step/sec: 933.892\n",
      "INFO:tensorflow:loss = 0.0003596757, step = 35600 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 913.428\n",
      "INFO:tensorflow:loss = 0.0007903095, step = 35700 (0.110 sec)\n",
      "INFO:tensorflow:global_step/sec: 946.445\n",
      "INFO:tensorflow:loss = 0.000612752, step = 35800 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 868.684\n",
      "INFO:tensorflow:loss = 0.00040744207, step = 35900 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 895.102\n",
      "INFO:tensorflow:loss = 0.0003478058, step = 36000 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 878.212\n",
      "INFO:tensorflow:loss = 0.0010372031, step = 36100 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 893.293\n",
      "INFO:tensorflow:loss = 0.00085367466, step = 36200 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 861.375\n",
      "INFO:tensorflow:loss = 0.0003116512, step = 36300 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 822.571\n",
      "INFO:tensorflow:loss = 0.0007065774, step = 36400 (0.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 919.894\n",
      "INFO:tensorflow:loss = 0.00018641839, step = 36500 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 969.979\n",
      "INFO:tensorflow:loss = 0.00027400532, step = 36600 (0.103 sec)\n",
      "INFO:tensorflow:global_step/sec: 925.286\n",
      "INFO:tensorflow:loss = 0.00037628383, step = 36700 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 930.199\n",
      "INFO:tensorflow:loss = 0.0007635035, step = 36800 (0.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 829.204\n",
      "INFO:tensorflow:loss = 0.00051770086, step = 36900 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 915.04\n",
      "INFO:tensorflow:loss = 0.0007371661, step = 37000 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 893.708\n",
      "INFO:tensorflow:loss = 0.00017992064, step = 37100 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 857.103\n",
      "INFO:tensorflow:loss = 0.00016092343, step = 37200 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 875.65\n",
      "INFO:tensorflow:loss = 0.0005725512, step = 37300 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 898.336\n",
      "INFO:tensorflow:loss = 0.00031324237, step = 37400 (0.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 951.869\n",
      "INFO:tensorflow:loss = 0.00037677237, step = 37500 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 845.885\n",
      "INFO:tensorflow:loss = 0.0002558426, step = 37600 (0.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 919.04\n",
      "INFO:tensorflow:loss = 8.1475155e-05, step = 37700 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 939.593\n",
      "INFO:tensorflow:loss = 0.0005693464, step = 37800 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 806.102\n",
      "INFO:tensorflow:loss = 0.0012520718, step = 37900 (0.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 913.742\n",
      "INFO:tensorflow:loss = 0.00022079474, step = 38000 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 877.312\n",
      "INFO:tensorflow:loss = 0.0008701265, step = 38100 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 874.349\n",
      "INFO:tensorflow:loss = 0.0007076465, step = 38200 (0.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 918.891\n",
      "INFO:tensorflow:loss = 4.5994268e-05, step = 38300 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 825.937\n",
      "INFO:tensorflow:loss = 0.000118618824, step = 38400 (0.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 923.512\n",
      "INFO:tensorflow:loss = 0.0005281278, step = 38500 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 919.368\n",
      "INFO:tensorflow:loss = 0.0007507463, step = 38600 (0.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 855.914\n",
      "INFO:tensorflow:loss = 0.0004117098, step = 38700 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 915.601\n",
      "INFO:tensorflow:loss = 0.00012507637, step = 38800 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 941.732\n",
      "INFO:tensorflow:loss = 0.0013767242, step = 38901 (0.106 sec)\n",
      "INFO:tensorflow:global_step/sec: 815.49\n",
      "INFO:tensorflow:loss = 0.00012228907, step = 39000 (0.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 892.162\n",
      "INFO:tensorflow:loss = 0.0006957, step = 39100 (0.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 870.701\n",
      "INFO:tensorflow:loss = 0.00037625982, step = 39200 (0.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 916.259\n",
      "INFO:tensorflow:loss = 0.0004538384, step = 39300 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 861.482\n",
      "INFO:tensorflow:loss = 0.00042230406, step = 39400 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 958.549\n",
      "INFO:tensorflow:loss = 0.00016296416, step = 39500 (0.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 939.704\n",
      "INFO:tensorflow:loss = 0.0006522428, step = 39600 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 833.823\n",
      "INFO:tensorflow:loss = 0.00017440172, step = 39700 (0.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 765.102\n",
      "INFO:tensorflow:loss = 0.0012321341, step = 39800 (0.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 861.574\n",
      "INFO:tensorflow:loss = 0.0007483138, step = 39900 (0.115 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40000 into /tmp/tmpvld1kthq/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0003995522.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SKCompat()"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = tf.contrib.learn.RunConfig(tf_random_seed=42) # 책에는 없음\n",
    "\n",
    "feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
    "dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                         feature_columns=feature_cols, config=config)\n",
    "dnn_clf = tf.contrib.learn.SKCompat(dnn_clf) # if TensorFlow >= 1.1\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "dnn_clf.fit(X_train, y_train, batch_size=50, steps=40000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpvld1kthq/model.ckpt-40000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9838"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = dnn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred['classes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07102101555478493"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "\n",
    "y_pred_proba = y_pred['probabilities']\n",
    "log_loss(y_test, y_pred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 저수준의 TensorFlow API 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 배치 데이터 정확도: 0.9 검증 세트 정확도: 0.9146\n",
      "1 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.936\n",
      "2 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.945\n",
      "3 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9512\n",
      "4 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9558\n",
      "5 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9566\n",
      "6 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9612\n",
      "7 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.963\n",
      "8 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9652\n",
      "9 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.966\n",
      "10 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9688\n",
      "11 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.969\n",
      "12 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.967\n",
      "13 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9706\n",
      "14 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9714\n",
      "15 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9732\n",
      "16 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9736\n",
      "17 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9742\n",
      "18 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9746\n",
      "19 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9748\n",
      "20 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9752\n",
      "21 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9752\n",
      "22 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9764\n",
      "23 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9752\n",
      "24 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9772\n",
      "25 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.977\n",
      "26 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9778\n",
      "27 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9774\n",
      "28 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9754\n",
      "29 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9776\n",
      "30 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9756\n",
      "31 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9772\n",
      "32 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9772\n",
      "33 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.979\n",
      "34 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9784\n",
      "35 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9778\n",
      "36 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.978\n",
      "37 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9776\n",
      "38 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9792\n",
      "39 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9776\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"배치 데이터 정확도:\", acc_batch, \"검증 세트 정확도:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # 또는 save_path를 사용합니다\n",
    "    X_new_scaled = X_test[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "예측 클래스: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n",
      "진짜 클래스: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"예측 클래스:\", y_pred)\n",
    "print(\"진짜 클래스:\", y_test[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_graph_in_jupyter import show_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 21\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 37\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `neuron_layer()` 대신 `dense()` 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "    y_proba = tf.nn.softmax(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 배치 데이터 정확도: 0.9 검증 세트 정확도: 0.9024\n",
      "1 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9254\n",
      "2 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9372\n",
      "3 배치 데이터 정확도: 0.9 검증 세트 정확도: 0.9416\n",
      "4 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9472\n",
      "5 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9512\n",
      "6 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9548\n",
      "7 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.961\n",
      "8 배치 데이터 정확도: 0.96 검증 세트 정확도: 0.9622\n",
      "9 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9648\n",
      "10 배치 데이터 정확도: 0.92 검증 세트 정확도: 0.9654\n",
      "11 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9668\n",
      "12 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9682\n",
      "13 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9702\n",
      "14 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9696\n",
      "15 배치 데이터 정확도: 0.94 검증 세트 정확도: 0.9718\n",
      "16 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9728\n",
      "17 배치 데이터 정확도: 1.0 검증 세트 정확도: 0.9728\n",
      "18 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9746\n",
      "19 배치 데이터 정확도: 0.98 검증 세트 정확도: 0.9752\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})\n",
    "        print(epoch, \"배치 데이터 정확도:\", acc_batch, \"검증 세트 정확도:\", acc_valid)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.7224827313584268&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 22\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 39\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/Softmax&quot;\\n  op: &quot;Softmax&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.7224827313584268&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 정답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "부록 A 참조."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_깊은 다층 퍼셉트론을 MNIST 데이터셋에 훈련시키고 98% 정확도를 얻을 수 있는지 확인해보세요. 9장의 마지막 연습문제에서와 같이 모든 부가 기능을 추가해보세요(즉, 체크포인트를 저장하고, 중지되었을 때 마지막 체크포인트를 복원하고, 서머리를 추가하고, 텐서보드를 사용해 학습 곡선을 그려보세요)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저 심층 신경망을 만듭니다. 한가지 추가된 것 외에는 앞서 했던 것과 동일합니다. 텐서보드에서 학습 곡선을 볼 수 있도록 훈련하는 동안 손실과 정확도를 기록하는 `tf.summary.scalar()` 추가합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int32, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드 로그를 기록할 디렉토리를 정의합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드 로그를 작성하는 데 필요한 `FileWriter` 객체를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "잠시만요! 조기 종료를 구현하는 것이 좋겠죠? 이렇게 하려면 검증 세트가 필요합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_valid = mnist.validation.images\n",
    "# y_valid = mnist.validation.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크: 0 \t검증 세트 정확도: 90.240% \t손실: 0.35379\n",
      "에포크: 5 \t검증 세트 정확도: 95.120% \t손실: 0.17921\n",
      "에포크: 10 \t검증 세트 정확도: 96.560% \t손실: 0.12786\n",
      "에포크: 15 \t검증 세트 정확도: 97.180% \t손실: 0.10323\n",
      "에포크: 20 \t검증 세트 정확도: 97.500% \t손실: 0.09165\n",
      "에포크: 25 \t검증 세트 정확도: 97.600% \t손실: 0.08209\n",
      "에포크: 30 \t검증 세트 정확도: 97.740% \t손실: 0.07892\n",
      "에포크: 35 \t검증 세트 정확도: 97.780% \t손실: 0.07425\n",
      "에포크: 40 \t검증 세트 정확도: 97.820% \t손실: 0.07160\n",
      "에포크: 45 \t검증 세트 정확도: 98.060% \t손실: 0.06736\n",
      "에포크: 50 \t검증 세트 정확도: 98.020% \t손실: 0.06730\n",
      "에포크: 55 \t검증 세트 정확도: 98.020% \t손실: 0.06682\n",
      "에포크: 60 \t검증 세트 정확도: 98.000% \t손실: 0.06725\n",
      "에포크: 65 \t검증 세트 정확도: 98.160% \t손실: 0.06670\n",
      "에포크: 70 \t검증 세트 정확도: 98.180% \t손실: 0.06611\n",
      "에포크: 75 \t검증 세트 정확도: 98.100% \t손실: 0.06648\n",
      "에포크: 80 \t검증 세트 정확도: 98.140% \t손실: 0.06664\n",
      "에포크: 85 \t검증 세트 정확도: 98.240% \t손실: 0.06608\n",
      "에포크: 90 \t검증 세트 정확도: 98.180% \t손실: 0.06739\n",
      "에포크: 95 \t검증 세트 정확도: 98.160% \t손실: 0.06891\n",
      "에포크: 100 \t검증 세트 정확도: 98.200% \t손실: 0.06878\n",
      "에포크: 105 \t검증 세트 정확도: 98.200% \t손실: 0.07068\n",
      "에포크: 110 \t검증 세트 정확도: 98.180% \t손실: 0.07054\n",
      "에포크: 115 \t검증 세트 정확도: 98.280% \t손실: 0.07055\n",
      "에포크: 120 \t검증 세트 정확도: 98.260% \t손실: 0.07310\n",
      "에포크: 125 \t검증 세트 정확도: 98.240% \t손실: 0.07137\n",
      "조기 종료\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # 체크포인트 파일이 있으면 모델을 복원하고 에포크 숫자를 로드합니다\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"이전 훈련이 중지되었습니다. 에포크 {}에서 시작합니다\".format(start_epoch))\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"에포크:\", epoch,\n",
    "                  \"\\t검증 세트 정확도: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\t손실: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"조기 종료\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9798"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}