{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.5.5\n",
      "IPython 6.3.0\n",
      "\n",
      "numpy 1.14.2\n",
      "scipy 1.0.1\n",
      "matplotlib 2.2.2\n",
      "tensorflow 1.7.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,scipy,matplotlib,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**10장 – 인공 신경망 소개**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 10장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 퍼셉트론"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # 꽃잎 길이, 꽃잎 너비\n",
    "y = (iris.target == 0).astype(np.int)\n",
    "\n",
    "per_clf = Perceptron(max_iter=5, random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 26484 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Iris-Setosa 아님\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap)\n",
    "plt.xlabel(\"꽃잎 길이\", fontsize=14)\n",
    "plt.ylabel(\"꽃잎 너비\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 활성화 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 42216 characters]\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"스텝\")\n",
    "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"로지스틱\")\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"활성화 함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"도함수\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 19428 characters]\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"활성화 함수: 헤비사이드\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"활성화 함수: 시그모이드\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST를 위한 FNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tf.learn를 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)  # deprecated 경고 메세지를 출력하지 않기 위해 \n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = mnist.train.images\n",
    "X_test = mnist.test.images\n",
    "y_train = mnist.train.labels.astype(\"int\")\n",
    "y_test = mnist.test.labels.astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmp8zcl_r8x/model.ckpt.\n",
      "INFO:tensorflow:step = 0, loss = 2.2865593\n",
      "INFO:tensorflow:global_step/sec: 863.886\n",
      "INFO:tensorflow:step = 100, loss = 0.3356912 (0.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.28\n",
      "INFO:tensorflow:step = 200, loss = 0.27911144 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.12\n",
      "INFO:tensorflow:step = 300, loss = 0.41703296 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1032.67\n",
      "INFO:tensorflow:step = 400, loss = 0.23802823 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1076.59\n",
      "INFO:tensorflow:step = 500, loss = 0.24561256 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1045.52\n",
      "INFO:tensorflow:step = 600, loss = 0.078200944 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1078.94\n",
      "INFO:tensorflow:step = 700, loss = 0.1284843 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1040.84\n",
      "INFO:tensorflow:step = 800, loss = 0.26003644 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1071.3\n",
      "INFO:tensorflow:step = 900, loss = 0.104407206 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1038.91\n",
      "INFO:tensorflow:step = 1000, loss = 0.1895929 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1038.51\n",
      "INFO:tensorflow:step = 1100, loss = 0.19595338 (0.097 sec)\n",
      "INFO:tensorflow:global_step/sec: 1075.8\n",
      "INFO:tensorflow:step = 1200, loss = 0.18142521 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1064.11\n",
      "INFO:tensorflow:step = 1300, loss = 0.17464441 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1057.53\n",
      "INFO:tensorflow:step = 1400, loss = 0.09934827 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1057.76\n",
      "INFO:tensorflow:step = 1500, loss = 0.09750447 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1061.14\n",
      "INFO:tensorflow:step = 1600, loss = 0.17522484 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.7\n",
      "INFO:tensorflow:step = 1700, loss = 0.02980093 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1086.39\n",
      "INFO:tensorflow:step = 1800, loss = 0.15622471 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1064.44\n",
      "INFO:tensorflow:step = 1900, loss = 0.08783154 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1061.32\n",
      "INFO:tensorflow:step = 2000, loss = 0.085202 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1028.3\n",
      "INFO:tensorflow:step = 2100, loss = 0.027735986 (0.097 sec)\n",
      "INFO:tensorflow:global_step/sec: 1036.98\n",
      "INFO:tensorflow:step = 2200, loss = 0.038919766 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1077.8\n",
      "INFO:tensorflow:step = 2300, loss = 0.05979547 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1088.85\n",
      "INFO:tensorflow:step = 2400, loss = 0.040494952 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1074.89\n",
      "INFO:tensorflow:step = 2500, loss = 0.060064077 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1094.41\n",
      "INFO:tensorflow:step = 2600, loss = 0.040182363 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1062.75\n",
      "INFO:tensorflow:step = 2700, loss = 0.020589514 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1068.68\n",
      "INFO:tensorflow:step = 2800, loss = 0.07433957 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 945.869\n",
      "INFO:tensorflow:step = 2900, loss = 0.13938075 (0.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 259.638\n",
      "INFO:tensorflow:step = 3000, loss = 0.017360074 (0.385 sec)\n",
      "INFO:tensorflow:global_step/sec: 437.029\n",
      "INFO:tensorflow:step = 3100, loss = 0.044024955 (0.227 sec)\n",
      "INFO:tensorflow:global_step/sec: 502.521\n",
      "INFO:tensorflow:step = 3200, loss = 0.012790296 (0.199 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.121\n",
      "INFO:tensorflow:step = 3300, loss = 0.025715819 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.325\n",
      "INFO:tensorflow:step = 3400, loss = 0.20885624 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.79\n",
      "INFO:tensorflow:step = 3500, loss = 0.1235593 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.162\n",
      "INFO:tensorflow:step = 3600, loss = 0.14184065 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 491.601\n",
      "INFO:tensorflow:step = 3700, loss = 0.044350732 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.105\n",
      "INFO:tensorflow:step = 3800, loss = 0.004993714 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 489.003\n",
      "INFO:tensorflow:step = 3900, loss = 0.08235403 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.164\n",
      "INFO:tensorflow:step = 4000, loss = 0.145322 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.909\n",
      "INFO:tensorflow:step = 4100, loss = 0.041134648 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 488.933\n",
      "INFO:tensorflow:step = 4200, loss = 0.0763246 (0.207 sec)\n",
      "INFO:tensorflow:global_step/sec: 488.99\n",
      "INFO:tensorflow:step = 4300, loss = 0.18186215 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.628\n",
      "INFO:tensorflow:step = 4400, loss = 0.10710266 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 488.229\n",
      "INFO:tensorflow:step = 4500, loss = 0.024140714 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 499.757\n",
      "INFO:tensorflow:step = 4600, loss = 0.018299235 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.752\n",
      "INFO:tensorflow:step = 4700, loss = 0.009908109 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.685\n",
      "INFO:tensorflow:step = 4800, loss = 0.02420118 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.856\n",
      "INFO:tensorflow:step = 4900, loss = 0.06875574 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 491.925\n",
      "INFO:tensorflow:step = 5000, loss = 0.07467158 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.601\n",
      "INFO:tensorflow:step = 5100, loss = 0.012627095 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.611\n",
      "INFO:tensorflow:step = 5200, loss = 0.029552968 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.138\n",
      "INFO:tensorflow:step = 5300, loss = 0.07503815 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.801\n",
      "INFO:tensorflow:step = 5400, loss = 0.043091077 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 486.841\n",
      "INFO:tensorflow:step = 5500, loss = 0.02273614 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.934\n",
      "INFO:tensorflow:step = 5600, loss = 0.08304 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 490.957\n",
      "INFO:tensorflow:step = 5700, loss = 0.016318094 (0.206 sec)\n",
      "INFO:tensorflow:global_step/sec: 489.518\n",
      "INFO:tensorflow:step = 5800, loss = 0.008830213 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 489.154\n",
      "INFO:tensorflow:step = 5900, loss = 0.07141432 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.34\n",
      "INFO:tensorflow:step = 6000, loss = 0.11991158 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 491.574\n",
      "INFO:tensorflow:step = 6100, loss = 0.025020095 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 498.961\n",
      "INFO:tensorflow:step = 6200, loss = 0.013107666 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 320.458\n",
      "INFO:tensorflow:step = 6300, loss = 0.06744149 (0.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 496.993\n",
      "INFO:tensorflow:step = 6400, loss = 0.021239351 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 496.395\n",
      "INFO:tensorflow:step = 6500, loss = 0.021382056 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 486.417\n",
      "INFO:tensorflow:step = 6600, loss = 0.015199767 (0.206 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.162\n",
      "INFO:tensorflow:step = 6701, loss = 0.021910792 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 489.113\n",
      "INFO:tensorflow:step = 6800, loss = 0.006276162 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 489.937\n",
      "INFO:tensorflow:step = 6900, loss = 0.011961005 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.458\n",
      "INFO:tensorflow:step = 7000, loss = 0.018515075 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.585\n",
      "INFO:tensorflow:step = 7100, loss = 0.0061758338 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.846\n",
      "INFO:tensorflow:step = 7200, loss = 0.034041166 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.166\n",
      "INFO:tensorflow:step = 7300, loss = 0.0065359403 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.044\n",
      "INFO:tensorflow:step = 7400, loss = 0.024646308 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 490.678\n",
      "INFO:tensorflow:step = 7500, loss = 0.0033852905 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.17\n",
      "INFO:tensorflow:step = 7600, loss = 0.029081745 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 486.047\n",
      "INFO:tensorflow:step = 7700, loss = 0.012077052 (0.206 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.03\n",
      "INFO:tensorflow:step = 7800, loss = 0.0038078267 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.781\n",
      "INFO:tensorflow:step = 7900, loss = 0.008777273 (0.203 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 492.665\n",
      "INFO:tensorflow:step = 8000, loss = 0.0022812437 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 490.858\n",
      "INFO:tensorflow:step = 8100, loss = 0.020457556 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.424\n",
      "INFO:tensorflow:step = 8200, loss = 0.009188964 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.437\n",
      "INFO:tensorflow:step = 8300, loss = 0.033466786 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.615\n",
      "INFO:tensorflow:step = 8400, loss = 0.0062055946 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 491.411\n",
      "INFO:tensorflow:step = 8500, loss = 0.00449106 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 490.749\n",
      "INFO:tensorflow:step = 8600, loss = 0.006774945 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.47\n",
      "INFO:tensorflow:step = 8700, loss = 0.004599379 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 487.138\n",
      "INFO:tensorflow:step = 8800, loss = 0.011535081 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.775\n",
      "INFO:tensorflow:step = 8900, loss = 0.001534353 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 491.899\n",
      "INFO:tensorflow:step = 9000, loss = 0.012303539 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 491.661\n",
      "INFO:tensorflow:step = 9100, loss = 0.006867932 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.23\n",
      "INFO:tensorflow:step = 9200, loss = 0.0049369726 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 496.759\n",
      "INFO:tensorflow:step = 9300, loss = 0.011140406 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.632\n",
      "INFO:tensorflow:step = 9400, loss = 0.041955262 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 489.6\n",
      "INFO:tensorflow:step = 9500, loss = 0.0040190606 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.486\n",
      "INFO:tensorflow:step = 9600, loss = 0.008081989 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 486.903\n",
      "INFO:tensorflow:step = 9700, loss = 0.004410262 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 485.777\n",
      "INFO:tensorflow:step = 9800, loss = 0.0045679696 (0.206 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.103\n",
      "INFO:tensorflow:step = 9900, loss = 0.012070759 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.943\n",
      "INFO:tensorflow:step = 10000, loss = 0.010317959 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.04\n",
      "INFO:tensorflow:step = 10100, loss = 0.004358228 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.954\n",
      "INFO:tensorflow:step = 10201, loss = 0.011694147 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 493.307\n",
      "INFO:tensorflow:step = 10300, loss = 0.00981953 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.112\n",
      "INFO:tensorflow:step = 10400, loss = 0.011855687 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.119\n",
      "INFO:tensorflow:step = 10500, loss = 0.00368173 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.515\n",
      "INFO:tensorflow:step = 10600, loss = 0.0072805565 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 497.036\n",
      "INFO:tensorflow:step = 10700, loss = 0.016785474 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 487.01\n",
      "INFO:tensorflow:step = 10800, loss = 0.0034213597 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.408\n",
      "INFO:tensorflow:step = 10900, loss = 0.0022000393 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 485.633\n",
      "INFO:tensorflow:step = 11000, loss = 0.01847614 (0.206 sec)\n",
      "INFO:tensorflow:global_step/sec: 491.384\n",
      "INFO:tensorflow:step = 11100, loss = 0.005845434 (0.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 489.845\n",
      "INFO:tensorflow:step = 11200, loss = 0.0010787308 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 490.226\n",
      "INFO:tensorflow:step = 11300, loss = 0.008111071 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 490.341\n",
      "INFO:tensorflow:step = 11400, loss = 0.0042507695 (0.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 495.292\n",
      "INFO:tensorflow:step = 11500, loss = 0.010545137 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 393.178\n",
      "INFO:tensorflow:step = 11600, loss = 0.0017785345 (0.253 sec)\n",
      "INFO:tensorflow:global_step/sec: 1010.3\n",
      "INFO:tensorflow:step = 11700, loss = 0.004827726 (0.099 sec)\n",
      "INFO:tensorflow:global_step/sec: 1058.92\n",
      "INFO:tensorflow:step = 11800, loss = 0.0005530121 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1055.53\n",
      "INFO:tensorflow:step = 11901, loss = 0.003321196 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1093.57\n",
      "INFO:tensorflow:step = 12000, loss = 0.00020355554 (0.091 sec)\n",
      "INFO:tensorflow:global_step/sec: 1029.83\n",
      "INFO:tensorflow:step = 12100, loss = 0.002730293 (0.097 sec)\n",
      "INFO:tensorflow:global_step/sec: 1057.81\n",
      "INFO:tensorflow:step = 12200, loss = 0.004475274 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1054.17\n",
      "INFO:tensorflow:step = 12300, loss = 0.0072001386 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1042.65\n",
      "INFO:tensorflow:step = 12400, loss = 0.00078346604 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1054.78\n",
      "INFO:tensorflow:step = 12500, loss = 0.00158522 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1023.23\n",
      "INFO:tensorflow:step = 12600, loss = 0.002770971 (0.098 sec)\n",
      "INFO:tensorflow:global_step/sec: 1040.29\n",
      "INFO:tensorflow:step = 12700, loss = 0.003350375 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1049\n",
      "INFO:tensorflow:step = 12800, loss = 0.006340498 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1068.38\n",
      "INFO:tensorflow:step = 12900, loss = 0.0033729768 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1068.97\n",
      "INFO:tensorflow:step = 13000, loss = 0.0041062376 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1047.41\n",
      "INFO:tensorflow:step = 13100, loss = 0.0028054 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1069.84\n",
      "INFO:tensorflow:step = 13200, loss = 0.002554019 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1068.01\n",
      "INFO:tensorflow:step = 13300, loss = 0.005321065 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1062.78\n",
      "INFO:tensorflow:step = 13400, loss = 0.024051609 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1064.88\n",
      "INFO:tensorflow:step = 13500, loss = 0.01614561 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1072.02\n",
      "INFO:tensorflow:step = 13600, loss = 0.006959138 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1078.93\n",
      "INFO:tensorflow:step = 13700, loss = 0.0022501848 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1077.54\n",
      "INFO:tensorflow:step = 13800, loss = 0.006876124 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1074.94\n",
      "INFO:tensorflow:step = 13900, loss = 0.0077551566 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1083.17\n",
      "INFO:tensorflow:step = 14000, loss = 0.0016267023 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1071.82\n",
      "INFO:tensorflow:step = 14100, loss = 0.0051740324 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1070.26\n",
      "INFO:tensorflow:step = 14200, loss = 0.004138899 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1059.85\n",
      "INFO:tensorflow:step = 14300, loss = 0.00069480867 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1054.49\n",
      "INFO:tensorflow:step = 14400, loss = 0.00069469266 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1026.83\n",
      "INFO:tensorflow:step = 14500, loss = 0.0008811189 (0.097 sec)\n",
      "INFO:tensorflow:global_step/sec: 1072.95\n",
      "INFO:tensorflow:step = 14600, loss = 0.0047456045 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1073.34\n",
      "INFO:tensorflow:step = 14700, loss = 0.0009369363 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1049.94\n",
      "INFO:tensorflow:step = 14800, loss = 0.00084657205 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1082.76\n",
      "INFO:tensorflow:step = 14900, loss = 0.0022222048 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1075.65\n",
      "INFO:tensorflow:step = 15000, loss = 0.0005325824 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1061.3\n",
      "INFO:tensorflow:step = 15100, loss = 0.0019375534 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1085.34\n",
      "INFO:tensorflow:step = 15200, loss = 0.0019285983 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1047.84\n",
      "INFO:tensorflow:step = 15300, loss = 0.0032861996 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1071.93\n",
      "INFO:tensorflow:step = 15400, loss = 0.0016209792 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.97\n",
      "INFO:tensorflow:step = 15500, loss = 0.006433259 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1067.66\n",
      "INFO:tensorflow:step = 15600, loss = 0.0051828623 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1064.26\n",
      "INFO:tensorflow:step = 15700, loss = 0.009161328 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1064.59\n",
      "INFO:tensorflow:step = 15800, loss = 0.0009421749 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1067.68\n",
      "INFO:tensorflow:step = 15900, loss = 0.00045572035 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1074.14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:step = 16000, loss = 0.00942636 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1048.41\n",
      "INFO:tensorflow:step = 16100, loss = 0.0026536172 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1071.31\n",
      "INFO:tensorflow:step = 16200, loss = 0.00011650101 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1074.7\n",
      "INFO:tensorflow:step = 16300, loss = 0.0023070304 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1076.11\n",
      "INFO:tensorflow:step = 16400, loss = 0.0015706429 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 999.386\n",
      "INFO:tensorflow:step = 16500, loss = 0.0030385917 (0.100 sec)\n",
      "INFO:tensorflow:global_step/sec: 1076.4\n",
      "INFO:tensorflow:step = 16600, loss = 0.0052231513 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1065.07\n",
      "INFO:tensorflow:step = 16700, loss = 0.0013896876 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.96\n",
      "INFO:tensorflow:step = 16800, loss = 0.0023458495 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1051.05\n",
      "INFO:tensorflow:step = 16900, loss = 0.0031086523 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.97\n",
      "INFO:tensorflow:step = 17000, loss = 0.0043154717 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1070.43\n",
      "INFO:tensorflow:step = 17100, loss = 0.0011421383 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1055.17\n",
      "INFO:tensorflow:step = 17200, loss = 0.0020875344 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1047.48\n",
      "INFO:tensorflow:step = 17300, loss = 0.0013131879 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1070.2\n",
      "INFO:tensorflow:step = 17400, loss = 0.0015903583 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1078.72\n",
      "INFO:tensorflow:step = 17500, loss = 0.00080179155 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1045.67\n",
      "INFO:tensorflow:step = 17600, loss = 0.0008254336 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1085.95\n",
      "INFO:tensorflow:step = 17700, loss = 0.00087235257 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1062.84\n",
      "INFO:tensorflow:step = 17800, loss = 0.00041898424 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1083.14\n",
      "INFO:tensorflow:step = 17900, loss = 0.0030533434 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1051.89\n",
      "INFO:tensorflow:step = 18000, loss = 0.0005241388 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1061.62\n",
      "INFO:tensorflow:step = 18100, loss = 0.00043429158 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1002.07\n",
      "INFO:tensorflow:step = 18200, loss = 0.003417973 (0.100 sec)\n",
      "INFO:tensorflow:global_step/sec: 1031.46\n",
      "INFO:tensorflow:step = 18300, loss = 0.008346801 (0.097 sec)\n",
      "INFO:tensorflow:global_step/sec: 1040.87\n",
      "INFO:tensorflow:step = 18400, loss = 0.00351103 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1085.29\n",
      "INFO:tensorflow:step = 18500, loss = 0.004740935 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1076.88\n",
      "INFO:tensorflow:step = 18600, loss = 0.0033694783 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1052.63\n",
      "INFO:tensorflow:step = 18700, loss = 0.0011617661 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1062.49\n",
      "INFO:tensorflow:step = 18800, loss = 0.0015412268 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1044.51\n",
      "INFO:tensorflow:step = 18900, loss = 0.0033057164 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1083.96\n",
      "INFO:tensorflow:step = 19000, loss = 0.0013642017 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1077.46\n",
      "INFO:tensorflow:step = 19100, loss = 0.0007333273 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1058.37\n",
      "INFO:tensorflow:step = 19200, loss = 0.001000539 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1080.51\n",
      "INFO:tensorflow:step = 19300, loss = 0.008101446 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1085.11\n",
      "INFO:tensorflow:step = 19400, loss = 0.00048663892 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1065.17\n",
      "INFO:tensorflow:step = 19500, loss = 0.0019621209 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1043.66\n",
      "INFO:tensorflow:step = 19600, loss = 0.00030130104 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1059.3\n",
      "INFO:tensorflow:step = 19700, loss = 6.4550775e-05 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 368.46\n",
      "INFO:tensorflow:step = 19800, loss = 0.0005093598 (0.271 sec)\n",
      "INFO:tensorflow:global_step/sec: 771.562\n",
      "INFO:tensorflow:step = 19900, loss = 0.0018963325 (0.130 sec)\n",
      "INFO:tensorflow:global_step/sec: 947.268\n",
      "INFO:tensorflow:step = 20000, loss = 0.0018636376 (0.105 sec)\n",
      "INFO:tensorflow:global_step/sec: 1038.47\n",
      "INFO:tensorflow:step = 20100, loss = 0.0007698703 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1067.26\n",
      "INFO:tensorflow:step = 20200, loss = 0.0023625873 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1064.77\n",
      "INFO:tensorflow:step = 20300, loss = 0.0015481006 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1061.68\n",
      "INFO:tensorflow:step = 20400, loss = 0.00039133965 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1060.29\n",
      "INFO:tensorflow:step = 20500, loss = 0.00064606004 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1058.81\n",
      "INFO:tensorflow:step = 20600, loss = 0.0010019239 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1064.68\n",
      "INFO:tensorflow:step = 20700, loss = 0.0005570926 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1042.34\n",
      "INFO:tensorflow:step = 20800, loss = 0.0019330138 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1061.22\n",
      "INFO:tensorflow:step = 20900, loss = 0.0013292085 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.28\n",
      "INFO:tensorflow:step = 21000, loss = 0.0024169027 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1051.56\n",
      "INFO:tensorflow:step = 21100, loss = 0.000972579 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1083.62\n",
      "INFO:tensorflow:step = 21200, loss = 0.0059826775 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1065.72\n",
      "INFO:tensorflow:step = 21300, loss = 0.0008278198 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1067.77\n",
      "INFO:tensorflow:step = 21400, loss = 0.0022743035 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1042.82\n",
      "INFO:tensorflow:step = 21500, loss = 0.00020002121 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.35\n",
      "INFO:tensorflow:step = 21600, loss = 0.0011737073 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1084.96\n",
      "INFO:tensorflow:step = 21700, loss = 0.0009844917 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1083.2\n",
      "INFO:tensorflow:step = 21800, loss = 0.0010091359 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1082.46\n",
      "INFO:tensorflow:step = 21900, loss = 0.0009299058 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1052.55\n",
      "INFO:tensorflow:step = 22000, loss = 0.00020814716 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1027.94\n",
      "INFO:tensorflow:step = 22100, loss = 0.00037157655 (0.097 sec)\n",
      "INFO:tensorflow:global_step/sec: 1065.69\n",
      "INFO:tensorflow:step = 22200, loss = 0.0012616038 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1068.97\n",
      "INFO:tensorflow:step = 22300, loss = 0.0014135549 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1072.42\n",
      "INFO:tensorflow:step = 22400, loss = 0.0016570385 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.71\n",
      "INFO:tensorflow:step = 22500, loss = 0.0017846385 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1053.13\n",
      "INFO:tensorflow:step = 22600, loss = 0.0030775133 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1057.09\n",
      "INFO:tensorflow:step = 22700, loss = 0.0007085688 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1067.1\n",
      "INFO:tensorflow:step = 22800, loss = 0.0012052946 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1079.7\n",
      "INFO:tensorflow:step = 22900, loss = 0.0023999047 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1063.43\n",
      "INFO:tensorflow:step = 23000, loss = 0.000981222 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1070.84\n",
      "INFO:tensorflow:step = 23100, loss = 0.004680404 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1070.63\n",
      "INFO:tensorflow:step = 23200, loss = 0.0016255345 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1071.34\n",
      "INFO:tensorflow:step = 23300, loss = 0.002042681 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1073.56\n",
      "INFO:tensorflow:step = 23400, loss = 0.0007716965 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1075.02\n",
      "INFO:tensorflow:step = 23500, loss = 0.0006288367 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1086.83\n",
      "INFO:tensorflow:step = 23600, loss = 0.00055640173 (0.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 1062.41\n",
      "INFO:tensorflow:step = 23700, loss = 0.0003840005 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 1075.23\n",
      "INFO:tensorflow:step = 23800, loss = 0.0010925684 (0.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 1046.62\n",
      "INFO:tensorflow:step = 23900, loss = 0.0019912808 (0.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 1049.26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:step = 24000, loss = 0.0010918856 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1053.58\n",
      "INFO:tensorflow:step = 24100, loss = 0.00042165944 (0.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 1058.73\n",
      "INFO:tensorflow:step = 24200, loss = 0.0014987171 (0.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 336.158\n",
      "INFO:tensorflow:step = 24300, loss = 9.394567e-05 (0.299 sec)\n",
      "INFO:tensorflow:global_step/sec: 280.438\n",
      "INFO:tensorflow:step = 24400, loss = 0.0022215478 (0.356 sec)\n",
      "INFO:tensorflow:global_step/sec: 362.696\n",
      "INFO:tensorflow:step = 24500, loss = 0.00089175755 (0.276 sec)\n",
      "INFO:tensorflow:global_step/sec: 373.614\n",
      "INFO:tensorflow:step = 24600, loss = 0.0004652041 (0.267 sec)\n",
      "INFO:tensorflow:global_step/sec: 367.967\n",
      "INFO:tensorflow:step = 24700, loss = 0.0011954482 (0.272 sec)\n",
      "INFO:tensorflow:global_step/sec: 363.143\n",
      "INFO:tensorflow:step = 24800, loss = 0.0018050713 (0.275 sec)\n",
      "INFO:tensorflow:global_step/sec: 365.058\n",
      "INFO:tensorflow:step = 24900, loss = 0.0015404779 (0.274 sec)\n",
      "INFO:tensorflow:global_step/sec: 369.41\n",
      "INFO:tensorflow:step = 25000, loss = 0.00038033843 (0.271 sec)\n",
      "INFO:tensorflow:global_step/sec: 374.89\n",
      "INFO:tensorflow:step = 25100, loss = 0.00076735124 (0.266 sec)\n",
      "INFO:tensorflow:global_step/sec: 478.358\n",
      "INFO:tensorflow:step = 25200, loss = 0.002223873 (0.210 sec)\n",
      "INFO:tensorflow:global_step/sec: 285.774\n",
      "INFO:tensorflow:step = 25300, loss = 0.00021607855 (0.349 sec)\n",
      "INFO:tensorflow:global_step/sec: 492.27\n",
      "INFO:tensorflow:step = 25400, loss = 0.00042847916 (0.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 494.256\n",
      "INFO:tensorflow:step = 25500, loss = 0.0012450315 (0.202 sec)\n",
      "INFO:tensorflow:global_step/sec: 498.804\n",
      "INFO:tensorflow:step = 25600, loss = 0.0010208315 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 499.16\n",
      "INFO:tensorflow:step = 25700, loss = 0.0006354126 (0.200 sec)\n",
      "INFO:tensorflow:global_step/sec: 496.729\n",
      "INFO:tensorflow:step = 25800, loss = 0.0007536512 (0.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 536.167\n",
      "INFO:tensorflow:step = 25900, loss = 0.0016661346 (0.186 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.661\n",
      "INFO:tensorflow:step = 26000, loss = 8.789845e-05 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.136\n",
      "INFO:tensorflow:step = 26100, loss = 0.000678507 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.015\n",
      "INFO:tensorflow:step = 26200, loss = 0.00047823825 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 624.03\n",
      "INFO:tensorflow:step = 26300, loss = 0.0008497758 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.423\n",
      "INFO:tensorflow:step = 26400, loss = 0.0018592369 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.997\n",
      "INFO:tensorflow:step = 26500, loss = 0.0005076347 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.635\n",
      "INFO:tensorflow:step = 26600, loss = 0.00055386784 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.944\n",
      "INFO:tensorflow:step = 26700, loss = 9.478591e-06 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.29\n",
      "INFO:tensorflow:step = 26800, loss = 0.0005859056 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.775\n",
      "INFO:tensorflow:step = 26900, loss = 0.0006593503 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.388\n",
      "INFO:tensorflow:step = 27000, loss = 0.00075005146 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.213\n",
      "INFO:tensorflow:step = 27100, loss = 0.00036156492 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.672\n",
      "INFO:tensorflow:step = 27200, loss = 0.0006153999 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.512\n",
      "INFO:tensorflow:step = 27300, loss = 0.0017520696 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.335\n",
      "INFO:tensorflow:step = 27400, loss = 0.00019366044 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.461\n",
      "INFO:tensorflow:step = 27500, loss = 0.0014988403 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.579\n",
      "INFO:tensorflow:step = 27600, loss = 0.0010438836 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.487\n",
      "INFO:tensorflow:step = 27700, loss = 0.00021674809 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.887\n",
      "INFO:tensorflow:step = 27800, loss = 0.00016371139 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.106\n",
      "INFO:tensorflow:step = 27900, loss = 0.0005285848 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.231\n",
      "INFO:tensorflow:step = 28000, loss = 0.0013335779 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.475\n",
      "INFO:tensorflow:step = 28100, loss = 0.0004803243 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.246\n",
      "INFO:tensorflow:step = 28200, loss = 0.0010146223 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.372\n",
      "INFO:tensorflow:step = 28300, loss = 0.00078414916 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.534\n",
      "INFO:tensorflow:step = 28400, loss = 0.00067714794 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 624.954\n",
      "INFO:tensorflow:step = 28500, loss = 0.00054019905 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.233\n",
      "INFO:tensorflow:step = 28600, loss = 0.00050456246 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 604.774\n",
      "INFO:tensorflow:step = 28700, loss = 0.0011200437 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 610.311\n",
      "INFO:tensorflow:step = 28800, loss = 0.0010517534 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.355\n",
      "INFO:tensorflow:step = 28900, loss = 0.0003801365 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.555\n",
      "INFO:tensorflow:step = 29000, loss = 0.0019070138 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.004\n",
      "INFO:tensorflow:step = 29100, loss = 0.0013867094 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.702\n",
      "INFO:tensorflow:step = 29200, loss = 0.0016565545 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 626.449\n",
      "INFO:tensorflow:step = 29300, loss = 0.001527873 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.085\n",
      "INFO:tensorflow:step = 29400, loss = 0.000885842 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.089\n",
      "INFO:tensorflow:step = 29500, loss = 0.0013642318 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 627.326\n",
      "INFO:tensorflow:step = 29600, loss = 0.00037206282 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.662\n",
      "INFO:tensorflow:step = 29700, loss = 0.00024796784 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.559\n",
      "INFO:tensorflow:step = 29800, loss = 0.00056447333 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.673\n",
      "INFO:tensorflow:step = 29900, loss = 0.001093979 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.055\n",
      "INFO:tensorflow:step = 30000, loss = 5.420686e-05 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.632\n",
      "INFO:tensorflow:step = 30100, loss = 0.000509425 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.871\n",
      "INFO:tensorflow:step = 30200, loss = 0.0006290985 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.328\n",
      "INFO:tensorflow:step = 30300, loss = 0.0010371926 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.225\n",
      "INFO:tensorflow:step = 30400, loss = 0.0013669133 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.083\n",
      "INFO:tensorflow:step = 30500, loss = 0.00094042433 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.386\n",
      "INFO:tensorflow:step = 30600, loss = 0.0010983101 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.806\n",
      "INFO:tensorflow:step = 30700, loss = 0.0008043706 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.452\n",
      "INFO:tensorflow:step = 30800, loss = 0.001329205 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.912\n",
      "INFO:tensorflow:step = 30900, loss = 0.0011187225 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.205\n",
      "INFO:tensorflow:step = 31000, loss = 0.0013951735 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.403\n",
      "INFO:tensorflow:step = 31100, loss = 0.0009561365 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 597.844\n",
      "INFO:tensorflow:step = 31200, loss = 0.0003641638 (0.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.771\n",
      "INFO:tensorflow:step = 31300, loss = 0.00055779377 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.239\n",
      "INFO:tensorflow:step = 31400, loss = 0.0010604032 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.604\n",
      "INFO:tensorflow:step = 31500, loss = 0.00021942765 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.466\n",
      "INFO:tensorflow:step = 31600, loss = 0.0003054194 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.745\n",
      "INFO:tensorflow:step = 31700, loss = 0.00054757134 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.723\n",
      "INFO:tensorflow:step = 31800, loss = 0.0003231604 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.816\n",
      "INFO:tensorflow:step = 31900, loss = 0.0008270536 (0.163 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 619.826\n",
      "INFO:tensorflow:step = 32000, loss = 0.0004234694 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.875\n",
      "INFO:tensorflow:step = 32100, loss = 0.00046731954 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.644\n",
      "INFO:tensorflow:step = 32200, loss = 0.0010028902 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.198\n",
      "INFO:tensorflow:step = 32300, loss = 0.0006235332 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.671\n",
      "INFO:tensorflow:step = 32400, loss = 0.0001313892 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 610.968\n",
      "INFO:tensorflow:step = 32500, loss = 0.0005731428 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 602.53\n",
      "INFO:tensorflow:step = 32600, loss = 8.8608795e-05 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.987\n",
      "INFO:tensorflow:step = 32700, loss = 0.0010296677 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 626.562\n",
      "INFO:tensorflow:step = 32800, loss = 0.00080902444 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 626.13\n",
      "INFO:tensorflow:step = 32900, loss = 0.0006836796 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 608.562\n",
      "INFO:tensorflow:step = 33000, loss = 0.00058407267 (0.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.473\n",
      "INFO:tensorflow:step = 33100, loss = 0.0018694943 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.593\n",
      "INFO:tensorflow:step = 33200, loss = 0.0011542265 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.68\n",
      "INFO:tensorflow:step = 33300, loss = 0.0001946216 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.432\n",
      "INFO:tensorflow:step = 33400, loss = 0.0013239475 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.802\n",
      "INFO:tensorflow:step = 33500, loss = 0.0003310576 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.365\n",
      "INFO:tensorflow:step = 33600, loss = 0.0009073831 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.452\n",
      "INFO:tensorflow:step = 33700, loss = 0.00091927603 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.595\n",
      "INFO:tensorflow:step = 33800, loss = 0.00068725436 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.954\n",
      "INFO:tensorflow:step = 33900, loss = 0.0012097562 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.854\n",
      "INFO:tensorflow:step = 34000, loss = 0.00088348956 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 609.422\n",
      "INFO:tensorflow:step = 34100, loss = 0.00065734924 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.499\n",
      "INFO:tensorflow:step = 34200, loss = 0.0013021302 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.728\n",
      "INFO:tensorflow:step = 34300, loss = 0.0002550179 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.895\n",
      "INFO:tensorflow:step = 34400, loss = 0.000365116 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.94\n",
      "INFO:tensorflow:step = 34500, loss = 0.00051270885 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.872\n",
      "INFO:tensorflow:step = 34600, loss = 0.00031644368 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.932\n",
      "INFO:tensorflow:step = 34700, loss = 0.0013708802 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 602.273\n",
      "INFO:tensorflow:step = 34800, loss = 0.00050053035 (0.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.365\n",
      "INFO:tensorflow:step = 34900, loss = 0.00096054235 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.994\n",
      "INFO:tensorflow:step = 35000, loss = 0.00038545177 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 625.247\n",
      "INFO:tensorflow:step = 35100, loss = 0.0010326216 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.33\n",
      "INFO:tensorflow:step = 35200, loss = 0.00029595714 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.114\n",
      "INFO:tensorflow:step = 35300, loss = 0.00040303366 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 611.763\n",
      "INFO:tensorflow:step = 35400, loss = 0.0012578097 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.667\n",
      "INFO:tensorflow:step = 35500, loss = 0.0003297355 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.649\n",
      "INFO:tensorflow:step = 35600, loss = 0.00025858288 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 624.253\n",
      "INFO:tensorflow:step = 35701, loss = 0.00062887854 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.471\n",
      "INFO:tensorflow:step = 35800, loss = 0.00059113634 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.056\n",
      "INFO:tensorflow:step = 35900, loss = 0.00071175856 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.295\n",
      "INFO:tensorflow:step = 36001, loss = 0.00031694007 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.713\n",
      "INFO:tensorflow:step = 36100, loss = 0.0013310048 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.303\n",
      "INFO:tensorflow:step = 36200, loss = 0.00068119355 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 605.853\n",
      "INFO:tensorflow:step = 36300, loss = 0.00030532392 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.624\n",
      "INFO:tensorflow:step = 36400, loss = 0.00065413833 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.937\n",
      "INFO:tensorflow:step = 36500, loss = 0.00033657916 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.432\n",
      "INFO:tensorflow:step = 36600, loss = 0.00039314985 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.87\n",
      "INFO:tensorflow:step = 36701, loss = 0.00032779446 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.821\n",
      "INFO:tensorflow:step = 36800, loss = 0.0007773846 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.385\n",
      "INFO:tensorflow:step = 36900, loss = 0.00069945364 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.472\n",
      "INFO:tensorflow:step = 37000, loss = 0.0012571606 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 625.694\n",
      "INFO:tensorflow:step = 37100, loss = 0.00027467374 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.396\n",
      "INFO:tensorflow:step = 37200, loss = 0.0003416043 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.761\n",
      "INFO:tensorflow:step = 37300, loss = 0.0007066202 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.321\n",
      "INFO:tensorflow:step = 37400, loss = 0.0003081595 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 619.103\n",
      "INFO:tensorflow:step = 37500, loss = 0.00021623619 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.245\n",
      "INFO:tensorflow:step = 37600, loss = 0.000343732 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.691\n",
      "INFO:tensorflow:step = 37701, loss = 0.0001752602 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 621.966\n",
      "INFO:tensorflow:step = 37800, loss = 0.00038288627 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.984\n",
      "INFO:tensorflow:step = 37900, loss = 0.001292691 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.564\n",
      "INFO:tensorflow:step = 38000, loss = 0.00023390524 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.264\n",
      "INFO:tensorflow:step = 38101, loss = 0.0006124892 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 612.452\n",
      "INFO:tensorflow:step = 38200, loss = 0.0005818791 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.278\n",
      "INFO:tensorflow:step = 38300, loss = 0.00014702232 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.12\n",
      "INFO:tensorflow:step = 38400, loss = 0.00016399998 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 606.339\n",
      "INFO:tensorflow:step = 38500, loss = 0.00057400187 (0.165 sec)\n",
      "INFO:tensorflow:global_step/sec: 627.221\n",
      "INFO:tensorflow:step = 38600, loss = 0.0007375686 (0.159 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.839\n",
      "INFO:tensorflow:step = 38700, loss = 0.00052899786 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 623.75\n",
      "INFO:tensorflow:step = 38800, loss = 0.000111311994 (0.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.398\n",
      "INFO:tensorflow:step = 38900, loss = 0.0014300102 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 617.545\n",
      "INFO:tensorflow:step = 39000, loss = 9.1103095e-05 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 620.913\n",
      "INFO:tensorflow:step = 39100, loss = 0.00094451586 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.911\n",
      "INFO:tensorflow:step = 39200, loss = 0.00041764986 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.881\n",
      "INFO:tensorflow:step = 39300, loss = 0.000314845 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 618.187\n",
      "INFO:tensorflow:step = 39400, loss = 0.0006256502 (0.162 sec)\n",
      "INFO:tensorflow:global_step/sec: 622.969\n",
      "INFO:tensorflow:step = 39500, loss = 0.00018341541 (0.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 613.211\n",
      "INFO:tensorflow:step = 39600, loss = 0.00045592964 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 614.614\n",
      "INFO:tensorflow:step = 39700, loss = 0.00019703162 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 615.455\n",
      "INFO:tensorflow:step = 39800, loss = 0.0012410291 (0.163 sec)\n",
      "INFO:tensorflow:global_step/sec: 616.796\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:step = 39900, loss = 0.00077618135 (0.162 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40000 into /tmp/tmp8zcl_r8x/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0004767616.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SKCompat()"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "config = tf.contrib.learn.RunConfig(tf_random_seed=42) # 책에는 없음\n",
    "\n",
    "feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
    "dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                         feature_columns=feature_cols, config=config)\n",
    "dnn_clf = tf.contrib.learn.SKCompat(dnn_clf) # if TensorFlow >= 1.1\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "dnn_clf.fit(X_train, y_train, batch_size=50, steps=40000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/tmpupshz99y/model.ckpt-40000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9823"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = dnn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred['classes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06889318829651588"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "\n",
    "y_pred_proba = y_pred['probabilities']\n",
    "log_loss(y_test, y_pred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 저수준의 TensorFlow API 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 정확도: 0.88 검증 세트 정확도: 0.9182\n",
      "1 훈련 정확도: 0.92 검증 세트 정확도: 0.9338\n",
      "2 훈련 정확도: 0.92 검증 세트 정확도: 0.9412\n",
      "3 훈련 정확도: 0.96 검증 세트 정확도: 0.9484\n",
      "4 훈련 정확도: 0.92 검증 세트 정확도: 0.9536\n",
      "5 훈련 정확도: 0.94 검증 세트 정확도: 0.9584\n",
      "6 훈련 정확도: 0.96 검증 세트 정확도: 0.9596\n",
      "7 훈련 정확도: 0.96 검증 세트 정확도: 0.9634\n",
      "8 훈련 정확도: 0.94 검증 세트 정확도: 0.9658\n",
      "9 훈련 정확도: 0.98 검증 세트 정확도: 0.9676\n",
      "10 훈련 정확도: 1.0 검증 세트 정확도: 0.9694\n",
      "11 훈련 정확도: 0.9 검증 세트 정확도: 0.9714\n",
      "12 훈련 정확도: 0.98 검증 세트 정확도: 0.9718\n",
      "13 훈련 정확도: 0.98 검증 세트 정확도: 0.9708\n",
      "14 훈련 정확도: 1.0 검증 세트 정확도: 0.9722\n",
      "15 훈련 정확도: 1.0 검증 세트 정확도: 0.9724\n",
      "16 훈련 정확도: 1.0 검증 세트 정확도: 0.9734\n",
      "17 훈련 정확도: 0.96 검증 세트 정확도: 0.9742\n",
      "18 훈련 정확도: 1.0 검증 세트 정확도: 0.9732\n",
      "19 훈련 정확도: 1.0 검증 세트 정확도: 0.9738\n",
      "20 훈련 정확도: 1.0 검증 세트 정확도: 0.976\n",
      "21 훈련 정확도: 1.0 검증 세트 정확도: 0.9756\n",
      "22 훈련 정확도: 1.0 검증 세트 정확도: 0.9758\n",
      "23 훈련 정확도: 1.0 검증 세트 정확도: 0.9744\n",
      "24 훈련 정확도: 1.0 검증 세트 정확도: 0.9752\n",
      "25 훈련 정확도: 1.0 검증 세트 정확도: 0.976\n",
      "26 훈련 정확도: 0.98 검증 세트 정확도: 0.9764\n",
      "27 훈련 정확도: 1.0 검증 세트 정확도: 0.9752\n",
      "28 훈련 정확도: 0.98 검증 세트 정확도: 0.9764\n",
      "29 훈련 정확도: 1.0 검증 세트 정확도: 0.9768\n",
      "30 훈련 정확도: 1.0 검증 세트 정확도: 0.9774\n",
      "31 훈련 정확도: 1.0 검증 세트 정확도: 0.9764\n",
      "32 훈련 정확도: 0.98 검증 세트 정확도: 0.978\n",
      "33 훈련 정확도: 0.98 검증 세트 정확도: 0.9782\n",
      "34 훈련 정확도: 0.98 검증 세트 정확도: 0.9782\n",
      "35 훈련 정확도: 1.0 검증 세트 정확도: 0.9784\n",
      "36 훈련 정확도: 0.98 검증 세트 정확도: 0.9788\n",
      "37 훈련 정확도: 0.98 검증 세트 정확도: 0.978\n",
      "38 훈련 정확도: 1.0 검증 세트 정확도: 0.979\n",
      "39 훈련 정확도: 1.0 검증 세트 정확도: 0.9792\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images,\n",
    "                                            y: mnist.validation.labels})\n",
    "        print(epoch, \"훈련 정확도:\", acc_train, \"검증 세트 정확도:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # 또는 save_path를 사용합니다\n",
    "    X_new_scaled = mnist.test.images[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "예측 클래스: [7 2 1 0 4 1 4 9 6 9 0 6 9 0 1 5 9 7 3 4]\n",
      "진짜 클래스: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"예측 클래스:\", y_pred)\n",
    "print(\"진짜 클래스:\", mnist.test.labels[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros/shape_as_tensor&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;dnn/hidden1/zeros/shape_as_tensor&quot;\\n  input: &quot;dnn/hidden1/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 23\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros/shape_as_tensor&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;dnn/hidden2/zeros/shape_as_tensor&quot;\\n  input: &quot;dnn/hidden2/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 41\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros/shape_as_tensor&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;dnn/outputs/zeros/shape_as_tensor&quot;\\n  input: &quot;dnn/outputs/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT64\\n        tensor_shape {\\n        }\\n        int64_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `neuron_layer()` 대신 `dense()` 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 정확도: 0.92 테스트 정확도: 0.9026\n",
      "1 훈련 정확도: 0.88 테스트 정확도: 0.9175\n",
      "2 훈련 정확도: 0.94 테스트 정확도: 0.9303\n",
      "3 훈련 정확도: 0.9 테스트 정확도: 0.9384\n",
      "4 훈련 정확도: 0.92 테스트 정확도: 0.9437\n",
      "5 훈련 정확도: 0.96 테스트 정확도: 0.948\n",
      "6 훈련 정확도: 0.92 테스트 정확도: 0.9517\n",
      "7 훈련 정확도: 1.0 테스트 정확도: 0.9535\n",
      "8 훈련 정확도: 0.96 테스트 정확도: 0.9554\n",
      "9 훈련 정확도: 0.94 테스트 정확도: 0.9588\n",
      "10 훈련 정확도: 0.94 테스트 정확도: 0.961\n",
      "11 훈련 정확도: 0.96 테스트 정확도: 0.9628\n",
      "12 훈련 정확도: 1.0 테스트 정확도: 0.9639\n",
      "13 훈련 정확도: 0.96 테스트 정확도: 0.9658\n",
      "14 훈련 정확도: 0.98 테스트 정확도: 0.9669\n",
      "15 훈련 정확도: 0.94 테스트 정확도: 0.968\n",
      "16 훈련 정확도: 0.92 테스트 정확도: 0.9688\n",
      "17 훈련 정확도: 0.98 테스트 정확도: 0.9691\n",
      "18 훈련 정확도: 1.0 테스트 정확도: 0.9703\n",
      "19 훈련 정확도: 1.0 테스트 정확도: 0.9702\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"훈련 정확도:\", acc_train, \"테스트 정확도:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.7224827313584268&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        unknown_rank: true\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 5\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros/shape_as_tensor&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros/shape_as_tensor&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 24\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros/shape_as_tensor&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros/shape_as_tensor&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 43\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros/shape_as_tensor&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;outputs/bias/Initializer/zeros/shape_as_tensor&quot;\\n  input: &quot;outputs/bias/Initializer/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT64\\n        tensor_shape {\\n        }\\n        int64_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/in_top_k/InTopKV2&quot;\\n  op: &quot;InTopKV2&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  input: &quot;eval/in_top_k/InTopKV2/k&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/in_top_k/InTopKV2&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2:2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2:3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2:4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2:5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.7224827313584268&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 정답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "부록 A 참조."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_깊은 다층 퍼셉트론을 MNIST 데이터셋에 훈련시키고 98% 정확도를 얻을 수 있는지 확인해보세요. 9장의 마지막 연습문제에서와 같이 모든 부가 기능을 추가해보세요(즉, 체크포인트를 저장하고, 중지되었을 때 마지막 체크포인트를 복원하고, 서머리를 추가하고, 텐서보드를 사용해 학습 곡선을 그려보세요)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저 심층 신경망을 만듭니다. 한가지 추가된 것 외에는 앞서 했던 것과 동일합니다. 텐서보드에서 학습 곡선을 볼 수 있도록 훈련하는 동안 손실과 정확도를 기록하는 `tf.summary.scalar()` 추가합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드 로그를 기록할 디렉토리를 정의합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드 로그를 작성하는 데 필요한 `FileWriter` 객체를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "잠시만요! 조기 종료를 구현하는 것이 좋겠죠? 이렇게 하려면 검증 세트가 필요합니다. 다행히 텐서플로의 `input_data()` 함수에서 반환되는 데이터셋은 이미 훈련 세트(이미 섞여있는 60,000개의 샘플), 검증 세트(5,000개의 샘플), 테스트 세트(5,000개의 샘플)로 나뉘어져 있습니다. 손쉽게 `X_valid`와 `y_valid`를 정의할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_valid = mnist.validation.images\n",
    "y_valid = mnist.validation.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크: 0 \t검증 세트 정확도: 90.540% \t손실: 0.35307\n",
      "에포크: 5 \t검증 세트 정확도: 95.040% \t손실: 0.17874\n",
      "에포크: 10 \t검증 세트 정확도: 96.620% \t손실: 0.12884\n",
      "에포크: 15 \t검증 세트 정확도: 97.120% \t손실: 0.10528\n",
      "에포크: 20 \t검증 세트 정확도: 97.480% \t손실: 0.09091\n",
      "에포크: 25 \t검증 세트 정확도: 97.740% \t손실: 0.08149\n",
      "에포크: 30 \t검증 세트 정확도: 97.860% \t손실: 0.07682\n",
      "에포크: 35 \t검증 세트 정확도: 97.940% \t손실: 0.07219\n",
      "에포크: 40 \t검증 세트 정확도: 98.100% \t손실: 0.07086\n",
      "에포크: 45 \t검증 세트 정확도: 98.080% \t손실: 0.06832\n",
      "에포크: 50 \t검증 세트 정확도: 98.100% \t손실: 0.06627\n",
      "에포크: 55 \t검증 세트 정확도: 98.140% \t손실: 0.06754\n",
      "에포크: 60 \t검증 세트 정확도: 98.200% \t손실: 0.06616\n",
      "에포크: 65 \t검증 세트 정확도: 98.060% \t손실: 0.06676\n",
      "에포크: 70 \t검증 세트 정확도: 98.120% \t손실: 0.06751\n",
      "에포크: 75 \t검증 세트 정확도: 98.140% \t손실: 0.06712\n",
      "에포크: 80 \t검증 세트 정확도: 98.200% \t손실: 0.06759\n",
      "에포크: 85 \t검증 세트 정확도: 98.160% \t손실: 0.06789\n",
      "에포크: 90 \t검증 세트 정확도: 98.220% \t손실: 0.06897\n",
      "에포크: 95 \t검증 세트 정확도: 98.240% \t손실: 0.07033\n",
      "에포크: 100 \t검증 세트 정확도: 98.260% \t손실: 0.06976\n",
      "에포크: 105 \t검증 세트 정확도: 98.240% \t손실: 0.07060\n",
      "에포크: 110 \t검증 세트 정확도: 98.320% \t손실: 0.07099\n",
      "조기 종료\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # 체크포인트 파일이 있으면 모델을 복원하고 에포크 숫자를 로드합니다\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"이전 훈련이 중지되었습니다. 에포크 {}에서 시작합니다\".format(start_epoch))\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"에포크:\", epoch,\n",
    "                  \"\\t검증 세트 정확도: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\t손실: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"조기 종료\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
