{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.5.5\n",
      "IPython 6.3.0\n",
      "\n",
      "numpy 1.14.3\n",
      "sklearn 0.19.1\n",
      "scipy 1.0.1\n",
      "matplotlib 2.2.2\n",
      "tensorflow 1.8.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,sklearn,scipy,matplotlib,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**14장 – 순환 신경망**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 14장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"rnn\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서플로를 임포트합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 기본 RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 수동으로 RNN 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 3\n",
    "n_neurons = 5\n",
    "\n",
    "X0 = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "X1 = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "Wx = tf.Variable(tf.random_normal(shape=[n_inputs, n_neurons],dtype=tf.float32))\n",
    "Wy = tf.Variable(tf.random_normal(shape=[n_neurons,n_neurons],dtype=tf.float32))\n",
    "b = tf.Variable(tf.zeros([1, n_neurons], dtype=tf.float32))\n",
    "\n",
    "Y0 = tf.tanh(tf.matmul(X0, Wx) + b)\n",
    "Y1 = tf.tanh(tf.matmul(Y0, Wy) + tf.matmul(X1, Wx) + b)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]]) # t = 0\n",
    "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]]) # t = 1\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.06640061  0.9625767   0.6810579   0.7091854  -0.89821595]\n",
      " [ 0.99777555 -0.7197888  -0.99657613  0.96739244 -0.99989706]\n",
      " [ 0.99999785 -0.9989881  -0.99999887  0.9967763  -0.9999999 ]\n",
      " [ 1.         -1.         -1.         -0.9981892   0.9995087 ]]\n"
     ]
    }
   ],
   "source": [
    "print(Y0_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.         -1.         -1.          0.40200272 -0.99999994]\n",
      " [-0.12210429  0.62805295  0.96718436 -0.9937122  -0.2583933 ]\n",
      " [ 0.99999815 -0.9999994  -0.99999744 -0.8594331  -0.99998796]\n",
      " [ 0.99928296 -0.9999981  -0.9999059   0.98579615 -0.9220575 ]]\n"
     ]
    }
   ],
   "source": [
    "print(Y1_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `static_rnn()`을 사용하여 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 3\n",
    "n_neurons = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X0 = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "X1 = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n",
    "output_seqs, states = tf.contrib.rnn.static_rnn(basic_cell, [X0, X1],\n",
    "                                                dtype=tf.float32)\n",
    "Y0, Y1 = output_seqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X0_batch = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 0, 1]])\n",
    "X1_batch = np.array([[9, 8, 7], [0, 0, 0], [6, 5, 4], [3, 2, 1]])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    Y0_val, Y1_val = sess.run([Y0, Y1], feed_dict={X0: X0_batch, X1: X1_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.30741337, -0.32884312, -0.6542847 , -0.9385059 ,  0.52089024],\n",
       "       [ 0.9912275 , -0.95425415, -0.7518078 , -0.9995208 ,  0.98202336],\n",
       "       [ 0.99992675, -0.99783254, -0.82473516, -0.99999636,  0.99947786],\n",
       "       [ 0.9967709 , -0.6875061 ,  0.8419969 ,  0.93039113,  0.81206834]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y0_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.99998885, -0.9997606 , -0.06679297, -0.9999803 ,  0.99982214],\n",
       "       [-0.65249425, -0.5152086 , -0.37968948, -0.5922594 , -0.08968376],\n",
       "       [ 0.998624  , -0.99715203, -0.03308632, -0.9991566 ,  0.9932902 ],\n",
       "       [ 0.99681675, -0.9598194 ,  0.39660627, -0.8307605 ,  0.7967197 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y1_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = \"b<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.3745401188473625&quot;).pbtxt = 'node {\\n  name: &quot;Placeholder&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 3\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Placeholder_1&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 3\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice/stack&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice/stack_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice/stack_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice&quot;\\n  op: &quot;StridedSlice&quot;\\n  input: &quot;rnn/Shape&quot;\\n  input: &quot;rnn/strided_slice/stack&quot;\\n  input: &quot;rnn/strided_slice/stack_1&quot;\\n  input: &quot;rnn/strided_slice/stack_2&quot;\\n  attr {\\n    key: &quot;Index&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;begin_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;ellipsis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;end_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;new_axis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;shrink_axis_mask&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;rnn/strided_slice&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/concat/axis&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/concat&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/ExpandDims&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/Const&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/concat/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/concat&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims_1/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims_1&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;rnn/strided_slice&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/ExpandDims_1/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\010\\\\000\\\\000\\\\000\\\\005\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.6793662309646606\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.6793662309646606\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 20\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 8\\n        }\\n        dim {\\n          size: 5\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 5\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 5\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/concat/axis&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/concat&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;Placeholder&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/zeros&quot;\\n  input: &quot;rnn/basic_rnn_cell/concat/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;rnn/basic_rnn_cell/concat&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;rnn/basic_rnn_cell/MatMul&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/Tanh&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;rnn/basic_rnn_cell/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/concat_1/axis&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/concat_1&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;Placeholder_1&quot;\\n  input: &quot;rnn/basic_rnn_cell/Tanh&quot;\\n  input: &quot;rnn/basic_rnn_cell/concat_1/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;rnn/basic_rnn_cell/concat_1&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/BiasAdd_1&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;rnn/basic_rnn_cell/MatMul_1&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/Tanh_1&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;rnn/basic_rnn_cell/BiasAdd_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^rnn/basic_rnn_cell/bias/Assign&quot;\\n  input: &quot;^rnn/basic_rnn_cell/kernel/Assign&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.3745401188473625&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 시퀀스 패딩"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_steps = 2\n",
    "n_inputs = 3\n",
    "n_neurons = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "X_seqs = tf.unstack(tf.transpose(X, perm=[1, 0, 2]))\n",
    "\n",
    "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n",
    "output_seqs, states = tf.contrib.rnn.static_rnn(basic_cell, X_seqs,\n",
    "                                                dtype=tf.float32)\n",
    "outputs = tf.transpose(tf.stack(output_seqs), perm=[1, 0, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_batch = np.array([\n",
    "        # t = 0      t = 1 \n",
    "        [[0, 1, 2], [9, 8, 7]], # 샘플 1\n",
    "        [[3, 4, 5], [0, 0, 0]], # 샘플 2\n",
    "        [[6, 7, 8], [6, 5, 4]], # 샘플 3\n",
    "        [[9, 0, 1], [3, 2, 1]], # 샘플 4\n",
    "    ])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    outputs_val = outputs.eval(feed_dict={X: X_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[-0.4565232  -0.6806412   0.40938237  0.631045   -0.45732823]\n",
      "  [-0.94288003 -0.9998869   0.9405581   0.99999845 -0.99999976]]\n",
      "\n",
      " [[-0.80015343 -0.99218273  0.78177965  0.9971032  -0.9964609 ]\n",
      "  [-0.637116    0.11300934  0.5798437   0.43105593 -0.6371699 ]]\n",
      "\n",
      " [[-0.93605185 -0.99983793  0.9308867   0.9999814  -0.99998313]\n",
      "  [-0.9165386  -0.99456036  0.89605415  0.9998719  -0.9999751 ]]\n",
      "\n",
      " [[ 0.99273676 -0.9981933  -0.5554365   0.99890316 -0.9953323 ]\n",
      "  [-0.02746333 -0.7319198   0.7827872   0.9525682  -0.9781772 ]]]\n"
     ]
    }
   ],
   "source": [
    "print(outputs_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.94288003 -0.9998869   0.9405581   0.99999845 -0.99999976]\n",
      " [-0.637116    0.11300934  0.5798437   0.43105593 -0.6371699 ]\n",
      " [-0.9165386  -0.99456036  0.89605415  0.9998719  -0.9999751 ]\n",
      " [-0.02746333 -0.7319198   0.7827872   0.9525682  -0.9781772 ]]\n"
     ]
    }
   ],
   "source": [
    "print(np.transpose(outputs_val, axes=[1, 0, 2])[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using `dynamic_rnn()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_steps = 2\n",
    "n_inputs = 3\n",
    "n_neurons = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "\n",
    "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n",
    "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_batch = np.array([\n",
    "        [[0, 1, 2], [9, 8, 7]], # instance 1\n",
    "        [[3, 4, 5], [0, 0, 0]], # instance 2\n",
    "        [[6, 7, 8], [6, 5, 4]], # instance 3\n",
    "        [[9, 0, 1], [3, 2, 1]], # instance 4\n",
    "    ])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    outputs_val = outputs.eval(feed_dict={X: X_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[-0.85115266  0.8735834   0.5802911   0.8954789  -0.0557505 ]\n",
      "  [-0.99999595  0.9999957   0.9981816   1.          0.37679613]]\n",
      "\n",
      " [[-0.99832934  0.9992038   0.98071444  0.99998486  0.25192654]\n",
      "  [-0.7081804  -0.07723369 -0.8522789   0.5845347  -0.7878094 ]]\n",
      "\n",
      " [[-0.9999826   0.9999953   0.99928635  1.          0.51590705]\n",
      "  [-0.9993956   0.9984095   0.83422637  0.9999998  -0.47325197]]\n",
      "\n",
      " [[ 0.87888587  0.07356028  0.9721692   0.9998546  -0.7351168 ]\n",
      "  [-0.9134514   0.3600957   0.7624865   0.99817705  0.80142   ]]]\n"
     ]
    }
   ],
   "source": [
    "print(outputs_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.3745401188473625&quot;).pbtxt = 'node {\\n  name: &quot;Placeholder&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 2\\n        }\\n        dim {\\n          size: 3\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Rank&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 3\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/range/start&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 2\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/range/delta&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/range&quot;\\n  op: &quot;Range&quot;\\n  input: &quot;rnn/range/start&quot;\\n  input: &quot;rnn/Rank&quot;\\n  input: &quot;rnn/range/delta&quot;\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat/values_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\001\\\\000\\\\000\\\\000\\\\000\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat/axis&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;rnn/concat/values_0&quot;\\n  input: &quot;rnn/range&quot;\\n  input: &quot;rnn/concat/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/transpose&quot;\\n  op: &quot;Transpose&quot;\\n  input: &quot;Placeholder&quot;\\n  input: &quot;rnn/concat&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tperm&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;rnn/transpose&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice/stack&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice/stack_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 2\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice/stack_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice&quot;\\n  op: &quot;StridedSlice&quot;\\n  input: &quot;rnn/Shape&quot;\\n  input: &quot;rnn/strided_slice/stack&quot;\\n  input: &quot;rnn/strided_slice/stack_1&quot;\\n  input: &quot;rnn/strided_slice/stack_2&quot;\\n  attr {\\n    key: &quot;Index&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;begin_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;ellipsis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;end_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;new_axis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;shrink_axis_mask&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;rnn/strided_slice&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/concat/axis&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/concat&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/ExpandDims&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/Const&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/concat/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/concat&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims_1/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/ExpandDims_1&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;rnn/strided_slice&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/ExpandDims_1/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/BasicRNNCellZeroState/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;rnn/transpose&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_1/stack&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_1/stack_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_1/stack_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_1&quot;\\n  op: &quot;StridedSlice&quot;\\n  input: &quot;rnn/Shape_1&quot;\\n  input: &quot;rnn/strided_slice_1/stack&quot;\\n  input: &quot;rnn/strided_slice_1/stack_1&quot;\\n  input: &quot;rnn/strided_slice_1/stack_2&quot;\\n  attr {\\n    key: &quot;Index&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;begin_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;ellipsis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;end_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;new_axis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;shrink_axis_mask&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Shape_2&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;rnn/transpose&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_2/stack&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_2/stack_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 2\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_2/stack_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/strided_slice_2&quot;\\n  op: &quot;StridedSlice&quot;\\n  input: &quot;rnn/Shape_2&quot;\\n  input: &quot;rnn/strided_slice_2/stack&quot;\\n  input: &quot;rnn/strided_slice_2/stack_1&quot;\\n  input: &quot;rnn/strided_slice_2/stack_2&quot;\\n  attr {\\n    key: &quot;Index&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;begin_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;ellipsis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;end_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;new_axis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;shrink_axis_mask&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;rnn/strided_slice_2&quot;\\n  input: &quot;rnn/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat_1/axis&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat_1&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;rnn/ExpandDims&quot;\\n  input: &quot;rnn/Const&quot;\\n  input: &quot;rnn/concat_1/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/zeros/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/zeros&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;rnn/concat_1&quot;\\n  input: &quot;rnn/zeros/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/time&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArray&quot;\\n  op: &quot;TensorArrayV3&quot;\\n  input: &quot;rnn/strided_slice_1&quot;\\n  attr {\\n    key: &quot;clear_after_read&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;dynamic_size&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;element_shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 5\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;identical_element_shapes&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;tensor_array_name&quot;\\n    value {\\n      s: &quot;rnn/dynamic_rnn/output_0&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArray_1&quot;\\n  op: &quot;TensorArrayV3&quot;\\n  input: &quot;rnn/strided_slice_1&quot;\\n  attr {\\n    key: &quot;clear_after_read&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;dynamic_size&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;element_shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 3\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;identical_element_shapes&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;tensor_array_name&quot;\\n    value {\\n      s: &quot;rnn/dynamic_rnn/input_0&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;rnn/transpose&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/strided_slice/stack&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/strided_slice/stack_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/strided_slice/stack_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/strided_slice&quot;\\n  op: &quot;StridedSlice&quot;\\n  input: &quot;rnn/TensorArrayUnstack/Shape&quot;\\n  input: &quot;rnn/TensorArrayUnstack/strided_slice/stack&quot;\\n  input: &quot;rnn/TensorArrayUnstack/strided_slice/stack_1&quot;\\n  input: &quot;rnn/TensorArrayUnstack/strided_slice/stack_2&quot;\\n  attr {\\n    key: &quot;Index&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;begin_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;ellipsis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;end_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;new_axis_mask&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;shrink_axis_mask&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/range/start&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/range/delta&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/range&quot;\\n  op: &quot;Range&quot;\\n  input: &quot;rnn/TensorArrayUnstack/range/start&quot;\\n  input: &quot;rnn/TensorArrayUnstack/strided_slice&quot;\\n  input: &quot;rnn/TensorArrayUnstack/range/delta&quot;\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3&quot;\\n  op: &quot;TensorArrayScatterV3&quot;\\n  input: &quot;rnn/TensorArray_1&quot;\\n  input: &quot;rnn/TensorArrayUnstack/range&quot;\\n  input: &quot;rnn/transpose&quot;\\n  input: &quot;rnn/TensorArray_1:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/transpose&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Maximum/x&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;rnn/Maximum/x&quot;\\n  input: &quot;rnn/strided_slice_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;rnn/strided_slice_1&quot;\\n  input: &quot;rnn/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/iteration_counter&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Enter&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/while/iteration_counter&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Enter_1&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/time&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Enter_2&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/TensorArray:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Enter_3&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/BasicRNNCellZeroState/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Merge&quot;\\n  op: &quot;Merge&quot;\\n  input: &quot;rnn/while/Enter&quot;\\n  input: &quot;rnn/while/NextIteration&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Merge_1&quot;\\n  op: &quot;Merge&quot;\\n  input: &quot;rnn/while/Enter_1&quot;\\n  input: &quot;rnn/while/NextIteration_1&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Merge_2&quot;\\n  op: &quot;Merge&quot;\\n  input: &quot;rnn/while/Enter_2&quot;\\n  input: &quot;rnn/while/NextIteration_2&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Merge_3&quot;\\n  op: &quot;Merge&quot;\\n  input: &quot;rnn/while/Enter_3&quot;\\n  input: &quot;rnn/while/NextIteration_3&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Less&quot;\\n  op: &quot;Less&quot;\\n  input: &quot;rnn/while/Merge&quot;\\n  input: &quot;rnn/while/Less/Enter&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Less/Enter&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/strided_slice_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Less_1&quot;\\n  op: &quot;Less&quot;\\n  input: &quot;rnn/while/Merge_1&quot;\\n  input: &quot;rnn/while/Less_1/Enter&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Less_1/Enter&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/Minimum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/LogicalAnd&quot;\\n  op: &quot;LogicalAnd&quot;\\n  input: &quot;rnn/while/Less&quot;\\n  input: &quot;rnn/while/Less_1&quot;\\n}\\nnode {\\n  name: &quot;rnn/while/LoopCond&quot;\\n  op: &quot;LoopCond&quot;\\n  input: &quot;rnn/while/LogicalAnd&quot;\\n}\\nnode {\\n  name: &quot;rnn/while/Switch&quot;\\n  op: &quot;Switch&quot;\\n  input: &quot;rnn/while/Merge&quot;\\n  input: &quot;rnn/while/LoopCond&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/while/Merge&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Switch_1&quot;\\n  op: &quot;Switch&quot;\\n  input: &quot;rnn/while/Merge_1&quot;\\n  input: &quot;rnn/while/LoopCond&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/while/Merge_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Switch_2&quot;\\n  op: &quot;Switch&quot;\\n  input: &quot;rnn/while/Merge_2&quot;\\n  input: &quot;rnn/while/LoopCond&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/while/Merge_2&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Switch_3&quot;\\n  op: &quot;Switch&quot;\\n  input: &quot;rnn/while/Merge_3&quot;\\n  input: &quot;rnn/while/LoopCond&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/while/Merge_3&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Identity&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/while/Switch:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Identity_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/while/Switch_1:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Identity_2&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/while/Switch_2:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Identity_3&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/while/Switch_3:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/add/y&quot;\\n  op: &quot;Const&quot;\\n  input: &quot;^rnn/while/Identity&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;rnn/while/Identity&quot;\\n  input: &quot;rnn/while/add/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/TensorArrayReadV3&quot;\\n  op: &quot;TensorArrayReadV3&quot;\\n  input: &quot;rnn/while/TensorArrayReadV3/Enter&quot;\\n  input: &quot;rnn/while/Identity_1&quot;\\n  input: &quot;rnn/while/TensorArrayReadV3/Enter_1&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/TensorArrayReadV3/Enter&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/TensorArray_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_RESOURCE\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/TensorArrayReadV3/Enter_1&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\010\\\\000\\\\000\\\\000\\\\005\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.6793662309646606\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.6793662309646606\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 87\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 8\\n        }\\n        dim {\\n          size: 5\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 5\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 5\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/basic_rnn_cell/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/basic_rnn_cell/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/basic_rnn_cell/concat/axis&quot;\\n  op: &quot;Const&quot;\\n  input: &quot;^rnn/while/Identity&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/basic_rnn_cell/concat&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;rnn/while/TensorArrayReadV3&quot;\\n  input: &quot;rnn/while/Identity_3&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/concat/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/basic_rnn_cell/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/concat&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/MatMul/Enter&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/basic_rnn_cell/MatMul/Enter&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/basic_rnn_cell/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/basic_rnn_cell/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/MatMul&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/BiasAdd/Enter&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/basic_rnn_cell/BiasAdd/Enter&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/basic_rnn_cell/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/basic_rnn_cell/Tanh&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/TensorArrayWrite/TensorArrayWriteV3&quot;\\n  op: &quot;TensorArrayWriteV3&quot;\\n  input: &quot;rnn/while/TensorArrayWrite/TensorArrayWriteV3/Enter&quot;\\n  input: &quot;rnn/while/Identity_1&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/Tanh&quot;\\n  input: &quot;rnn/while/Identity_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/while/basic_rnn_cell/Tanh&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/TensorArrayWrite/TensorArrayWriteV3/Enter&quot;\\n  op: &quot;Enter&quot;\\n  input: &quot;rnn/TensorArray&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_RESOURCE\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/while/basic_rnn_cell/Tanh&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;frame_name&quot;\\n    value {\\n      s: &quot;rnn/while/while_context&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;is_constant&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;parallel_iterations&quot;\\n    value {\\n      i: 32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/add_1/y&quot;\\n  op: &quot;Const&quot;\\n  input: &quot;^rnn/while/Identity&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/add_1&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;rnn/while/Identity_1&quot;\\n  input: &quot;rnn/while/add_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/NextIteration&quot;\\n  op: &quot;NextIteration&quot;\\n  input: &quot;rnn/while/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/NextIteration_1&quot;\\n  op: &quot;NextIteration&quot;\\n  input: &quot;rnn/while/add_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/NextIteration_2&quot;\\n  op: &quot;NextIteration&quot;\\n  input: &quot;rnn/while/TensorArrayWrite/TensorArrayWriteV3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/NextIteration_3&quot;\\n  op: &quot;NextIteration&quot;\\n  input: &quot;rnn/while/basic_rnn_cell/Tanh&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Exit&quot;\\n  op: &quot;Exit&quot;\\n  input: &quot;rnn/while/Switch&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Exit_1&quot;\\n  op: &quot;Exit&quot;\\n  input: &quot;rnn/while/Switch_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Exit_2&quot;\\n  op: &quot;Exit&quot;\\n  input: &quot;rnn/while/Switch_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/while/Exit_3&quot;\\n  op: &quot;Exit&quot;\\n  input: &quot;rnn/while/Switch_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayStack/TensorArraySizeV3&quot;\\n  op: &quot;TensorArraySizeV3&quot;\\n  input: &quot;rnn/TensorArray&quot;\\n  input: &quot;rnn/while/Exit_2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/TensorArray&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayStack/range/start&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/TensorArray&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayStack/range/delta&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/TensorArray&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayStack/range&quot;\\n  op: &quot;Range&quot;\\n  input: &quot;rnn/TensorArrayStack/range/start&quot;\\n  input: &quot;rnn/TensorArrayStack/TensorArraySizeV3&quot;\\n  input: &quot;rnn/TensorArrayStack/range/delta&quot;\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/TensorArray&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/TensorArrayStack/TensorArrayGatherV3&quot;\\n  op: &quot;TensorArrayGatherV3&quot;\\n  input: &quot;rnn/TensorArray&quot;\\n  input: &quot;rnn/TensorArrayStack/range&quot;\\n  input: &quot;rnn/while/Exit_2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@rnn/TensorArray&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;element_shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 5\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/Rank_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 3\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/range_1/start&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 2\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/range_1/delta&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/range_1&quot;\\n  op: &quot;Range&quot;\\n  input: &quot;rnn/range_1/start&quot;\\n  input: &quot;rnn/Rank_1&quot;\\n  input: &quot;rnn/range_1/delta&quot;\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat_2/values_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\001\\\\000\\\\000\\\\000\\\\000\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat_2/axis&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/concat_2&quot;\\n  op: &quot;ConcatV2&quot;\\n  input: &quot;rnn/concat_2/values_0&quot;\\n  input: &quot;rnn/range_1&quot;\\n  input: &quot;rnn/concat_2/axis&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;rnn/transpose_1&quot;\\n  op: &quot;Transpose&quot;\\n  input: &quot;rnn/TensorArrayStack/TensorArrayGatherV3&quot;\\n  input: &quot;rnn/concat_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tperm&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^rnn/basic_rnn_cell/bias/Assign&quot;\\n  input: &quot;^rnn/basic_rnn_cell/kernel/Assign&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.3745401188473625&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 시퀀스 길이 지정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_steps = 2\n",
    "n_inputs = 3\n",
    "n_neurons = 5\n",
    "\n",
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_length = tf.placeholder(tf.int32, [None])\n",
    "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32,\n",
    "                                    sequence_length=seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_batch = np.array([\n",
    "        # 스텝 0     스텝 1\n",
    "        [[0, 1, 2], [9, 8, 7]], # 샘플 1\n",
    "        [[3, 4, 5], [0, 0, 0]], # 샘플 2 (0 벡터로 패딩)\n",
    "        [[6, 7, 8], [6, 5, 4]], # 샘플 3\n",
    "        [[9, 0, 1], [3, 2, 1]], # 샘플 4\n",
    "    ])\n",
    "seq_length_batch = np.array([2, 1, 2, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    outputs_val, states_val = sess.run(\n",
    "        [outputs, states], feed_dict={X: X_batch, seq_length: seq_length_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[-0.91231877  0.16516447  0.5548655  -0.3915935   0.20846416]\n",
      "  [-1.          0.95672596  0.9983169   0.9997017   0.96518576]]\n",
      "\n",
      " [[-0.99986124  0.67022896  0.9723652   0.66310453  0.74457586]\n",
      "  [ 0.          0.          0.          0.          0.        ]]\n",
      "\n",
      " [[-0.99999976  0.89679974  0.9986295   0.96475154  0.93662006]\n",
      "  [-0.99995255  0.96819544  0.9600286   0.9870626   0.8545923 ]]\n",
      "\n",
      " [[-0.9643544   0.9950159  -0.361507    0.99833775  0.99949706]\n",
      "  [-0.9613586   0.9568762   0.71322876  0.9772921  -0.09582992]]]\n"
     ]
    }
   ],
   "source": [
    "print(outputs_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.          0.95672596  0.9983169   0.9997017   0.96518576]\n",
      " [-0.99986124  0.67022896  0.9723652   0.66310453  0.74457586]\n",
      " [-0.99995255  0.96819544  0.9600286   0.9870626   0.8545923 ]\n",
      " [-0.9613586   0.9568762   0.71322876  0.9772921  -0.09582992]]\n"
     ]
    }
   ],
   "source": [
    "print(states_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 시퀀스 분류기 훈련하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "n_neurons = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n",
    "outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)\n",
    "\n",
    "logits = tf.layers.dense(states, n_outputs)\n",
    "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                          logits=logits)\n",
    "loss = tf.reduce_mean(xentropy)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-36-9aff0e2313ad>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/handson-ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")\n",
    "X_test = mnist.test.images.reshape((-1, n_steps, n_inputs))\n",
    "y_test = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 정확도: 0.93333334 테스트 정확도: 0.9311\n",
      "1 훈련 정확도: 0.96666664 테스트 정확도: 0.9522\n",
      "2 훈련 정확도: 0.97333336 테스트 정확도: 0.9584\n",
      "3 훈련 정확도: 0.96 테스트 정확도: 0.966\n",
      "4 훈련 정확도: 0.97333336 테스트 정확도: 0.9685\n",
      "5 훈련 정확도: 0.98 테스트 정확도: 0.9711\n",
      "6 훈련 정확도: 0.96666664 테스트 정확도: 0.9687\n",
      "7 훈련 정확도: 0.97333336 테스트 정확도: 0.9687\n",
      "8 훈련 정확도: 0.96666664 테스트 정확도: 0.9718\n",
      "9 훈련 정확도: 0.98 테스트 정확도: 0.9694\n",
      "10 훈련 정확도: 0.9866667 테스트 정확도: 0.9736\n",
      "11 훈련 정확도: 0.96 테스트 정확도: 0.9702\n",
      "12 훈련 정확도: 0.9866667 테스트 정확도: 0.9771\n",
      "13 훈련 정확도: 0.96666664 테스트 정확도: 0.9695\n",
      "14 훈련 정확도: 0.97333336 테스트 정확도: 0.9706\n",
      "15 훈련 정확도: 0.99333334 테스트 정확도: 0.9763\n",
      "16 훈련 정확도: 0.9866667 테스트 정확도: 0.9778\n",
      "17 훈련 정확도: 0.9866667 테스트 정확도: 0.9752\n",
      "18 훈련 정확도: 0.9866667 테스트 정확도: 0.975\n",
      "19 훈련 정확도: 0.97333336 테스트 정확도: 0.9745\n",
      "20 훈련 정확도: 0.99333334 테스트 정확도: 0.9773\n",
      "21 훈련 정확도: 0.98 테스트 정확도: 0.98\n",
      "22 훈련 정확도: 0.98 테스트 정확도: 0.9794\n",
      "23 훈련 정확도: 0.98 테스트 정확도: 0.9739\n",
      "24 훈련 정확도: 0.98 테스트 정확도: 0.9757\n",
      "25 훈련 정확도: 1.0 테스트 정확도: 0.9719\n",
      "26 훈련 정확도: 0.97333336 테스트 정확도: 0.9803\n",
      "27 훈련 정확도: 1.0 테스트 정확도: 0.9796\n",
      "28 훈련 정확도: 0.96666664 테스트 정확도: 0.977\n",
      "29 훈련 정확도: 0.99333334 테스트 정확도: 0.9799\n",
      "30 훈련 정확도: 0.9866667 테스트 정확도: 0.9754\n",
      "31 훈련 정확도: 0.99333334 테스트 정확도: 0.9791\n",
      "32 훈련 정확도: 0.98 테스트 정확도: 0.9775\n",
      "33 훈련 정확도: 0.99333334 테스트 정확도: 0.9755\n",
      "34 훈련 정확도: 0.9866667 테스트 정확도: 0.9723\n",
      "35 훈련 정확도: 1.0 테스트 정확도: 0.9797\n",
      "36 훈련 정확도: 1.0 테스트 정확도: 0.9798\n",
      "37 훈련 정확도: 0.99333334 테스트 정확도: 0.9795\n",
      "38 훈련 정확도: 0.99333334 테스트 정확도: 0.9766\n",
      "39 훈련 정확도: 0.99333334 테스트 정확도: 0.9792\n",
      "40 훈련 정확도: 1.0 테스트 정확도: 0.9811\n",
      "41 훈련 정확도: 0.97333336 테스트 정확도: 0.9728\n",
      "42 훈련 정확도: 0.98 테스트 정확도: 0.981\n",
      "43 훈련 정확도: 0.99333334 테스트 정확도: 0.9735\n",
      "44 훈련 정확도: 0.9866667 테스트 정확도: 0.9754\n",
      "45 훈련 정확도: 0.9866667 테스트 정확도: 0.9787\n",
      "46 훈련 정확도: 0.9866667 테스트 정확도: 0.9822\n",
      "47 훈련 정확도: 1.0 테스트 정확도: 0.9777\n",
      "48 훈련 정확도: 0.9866667 테스트 정확도: 0.9762\n",
      "49 훈련 정확도: 1.0 테스트 정확도: 0.976\n",
      "50 훈련 정확도: 0.99333334 테스트 정확도: 0.9819\n",
      "51 훈련 정확도: 1.0 테스트 정확도: 0.9787\n",
      "52 훈련 정확도: 0.9866667 테스트 정확도: 0.9782\n",
      "53 훈련 정확도: 0.9866667 테스트 정확도: 0.9766\n",
      "54 훈련 정확도: 0.99333334 테스트 정확도: 0.981\n",
      "55 훈련 정확도: 0.99333334 테스트 정확도: 0.9789\n",
      "56 훈련 정확도: 0.99333334 테스트 정확도: 0.9806\n",
      "57 훈련 정확도: 1.0 테스트 정확도: 0.9835\n",
      "58 훈련 정확도: 0.9866667 테스트 정확도: 0.9789\n",
      "59 훈련 정확도: 1.0 테스트 정확도: 0.9738\n",
      "60 훈련 정확도: 0.99333334 테스트 정확도: 0.9702\n",
      "61 훈련 정확도: 1.0 테스트 정확도: 0.9803\n",
      "62 훈련 정확도: 1.0 테스트 정확도: 0.9766\n",
      "63 훈련 정확도: 0.9866667 테스트 정확도: 0.9797\n",
      "64 훈련 정확도: 0.9866667 테스트 정확도: 0.9758\n",
      "65 훈련 정확도: 0.9866667 테스트 정확도: 0.9764\n",
      "66 훈련 정확도: 0.9866667 테스트 정확도: 0.9773\n",
      "67 훈련 정확도: 0.99333334 테스트 정확도: 0.9811\n",
      "68 훈련 정확도: 0.99333334 테스트 정확도: 0.9721\n",
      "69 훈련 정확도: 0.9866667 테스트 정확도: 0.9795\n",
      "70 훈련 정확도: 0.98 테스트 정확도: 0.9754\n",
      "71 훈련 정확도: 1.0 테스트 정확도: 0.9786\n",
      "72 훈련 정확도: 0.9866667 테스트 정확도: 0.9771\n",
      "73 훈련 정확도: 1.0 테스트 정확도: 0.9795\n",
      "74 훈련 정확도: 0.99333334 테스트 정확도: 0.9805\n",
      "75 훈련 정확도: 0.97333336 테스트 정확도: 0.9807\n",
      "76 훈련 정확도: 0.97333336 테스트 정확도: 0.9752\n",
      "77 훈련 정확도: 1.0 테스트 정확도: 0.9808\n",
      "78 훈련 정확도: 0.99333334 테스트 정확도: 0.9732\n",
      "79 훈련 정확도: 0.99333334 테스트 정확도: 0.9805\n",
      "80 훈련 정확도: 0.9866667 테스트 정확도: 0.9779\n",
      "81 훈련 정확도: 0.99333334 테스트 정확도: 0.9791\n",
      "82 훈련 정확도: 0.99333334 테스트 정확도: 0.9747\n",
      "83 훈련 정확도: 0.99333334 테스트 정확도: 0.9784\n",
      "84 훈련 정확도: 0.99333334 테스트 정확도: 0.9768\n",
      "85 훈련 정확도: 0.97333336 테스트 정확도: 0.9733\n",
      "86 훈련 정확도: 0.99333334 테스트 정확도: 0.9801\n",
      "87 훈련 정확도: 0.99333334 테스트 정확도: 0.9794\n",
      "88 훈련 정확도: 0.99333334 테스트 정확도: 0.9772\n",
      "89 훈련 정확도: 0.99333334 테스트 정확도: 0.9773\n",
      "90 훈련 정확도: 1.0 테스트 정확도: 0.9779\n",
      "91 훈련 정확도: 0.99333334 테스트 정확도: 0.9761\n",
      "92 훈련 정확도: 0.97333336 테스트 정확도: 0.9753\n",
      "93 훈련 정확도: 1.0 테스트 정확도: 0.975\n",
      "94 훈련 정확도: 1.0 테스트 정확도: 0.9802\n",
      "95 훈련 정확도: 0.99333334 테스트 정확도: 0.9797\n",
      "96 훈련 정확도: 0.9866667 테스트 정확도: 0.9796\n",
      "97 훈련 정확도: 1.0 테스트 정확도: 0.9783\n",
      "98 훈련 정확도: 0.9866667 테스트 정확도: 0.9772\n",
      "99 훈련 정확도: 1.0 테스트 정확도: 0.9793\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(epoch, \"훈련 정확도:\", acc_train, \"테스트 정확도:\", acc_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 다층 RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.int32, [None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_neurons = 100\n",
    "n_layers = 3\n",
    "\n",
    "layers = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons,\n",
    "                                      activation=tf.nn.relu)\n",
    "          for layer in range(n_layers)]\n",
    "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(layers)\n",
    "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "states_concat = tf.concat(axis=1, values=states)\n",
    "logits = tf.layers.dense(states_concat, n_outputs)\n",
    "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "loss = tf.reduce_mean(xentropy)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 정확도: 0.93333334 테스트 정확도: 0.939\n",
      "1 훈련 정확도: 0.98 테스트 정확도: 0.9589\n",
      "2 훈련 정확도: 0.94666666 테스트 정확도: 0.9677\n",
      "3 훈련 정확도: 0.98 테스트 정확도: 0.9756\n",
      "4 훈련 정확도: 0.98 테스트 정확도: 0.9765\n",
      "5 훈련 정확도: 0.9866667 테스트 정확도: 0.9741\n",
      "6 훈련 정확도: 0.97333336 테스트 정확도: 0.9783\n",
      "7 훈련 정확도: 1.0 테스트 정확도: 0.9804\n",
      "8 훈련 정확도: 0.99333334 테스트 정확도: 0.979\n",
      "9 훈련 정확도: 0.9866667 테스트 정확도: 0.9815\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            X_batch = X_batch.reshape((-1, n_steps, n_inputs))\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(epoch, \"훈련 정확도:\", acc_train, \"테스트 정확도:\", acc_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 시계열"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_min, t_max = 0, 30\n",
    "resolution = 0.1\n",
    "\n",
    "def time_series(t):\n",
    "    return t * np.sin(t) / 3 + 2 * np.sin(t*5)\n",
    "\n",
    "def next_batch(batch_size, n_steps):\n",
    "    t0 = np.random.rand(batch_size, 1) * (t_max - t_min - n_steps * resolution)\n",
    "    Ts = t0 + np.arange(0., n_steps + 1) * resolution\n",
    "    ys = time_series(Ts)\n",
    "    return ys[:, :-1].reshape(-1, n_steps, 1), ys[:, 1:].reshape(-1, n_steps, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 45296 characters]\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = np.linspace(t_min, t_max, int((t_max - t_min) / resolution))\n",
    "\n",
    "n_steps = 20\n",
    "t_instance = np.linspace(12.2, 12.2 + resolution * (n_steps + 1), n_steps + 1)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "plt.subplot(121)\n",
    "plt.title(\"시계열 데이터 (인공 생성)\", fontsize=14)\n",
    "plt.plot(t, time_series(t), label=r\"$t . \\sin(t) / 3 + 2 . \\sin(5t)$\")\n",
    "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"b-\", linewidth=3, label=\"훈련 샘플\")\n",
    "plt.legend(loc=\"lower left\", fontsize=14)\n",
    "plt.axis([0, 30, -17, 13])\n",
    "plt.xlabel(\"시간\")\n",
    "plt.ylabel(\"값\", rotation=0)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.title(\"훈련 샘플\", fontsize=14)\n",
    "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=12, label=\"샘플\")\n",
    "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markeredgewidth=0.5, markeredgecolor=\"b\", markersize=14, label=\"타깃\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"시간\")\n",
    "\n",
    "\n",
    "save_fig(\"time_series_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_batch, y_batch = next_batch(1, n_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.38452097,  2.05081182],\n",
       "       [ 2.05081182,  2.29742291],\n",
       "       [ 2.29742291,  2.0465599 ],\n",
       "       [ 2.0465599 ,  1.34009916],\n",
       "       [ 1.34009916,  0.32948704],\n",
       "       [ 0.32948704, -0.76115235],\n",
       "       [-0.76115235, -1.68967022],\n",
       "       [-1.68967022, -2.25492776],\n",
       "       [-2.25492776, -2.34576159],\n",
       "       [-2.34576159, -1.96789418],\n",
       "       [-1.96789418, -1.24220428],\n",
       "       [-1.24220428, -0.37478448],\n",
       "       [-0.37478448,  0.39387907],\n",
       "       [ 0.39387907,  0.84815766],\n",
       "       [ 0.84815766,  0.85045064],\n",
       "       [ 0.85045064,  0.3752526 ],\n",
       "       [ 0.3752526 , -0.48422846],\n",
       "       [-0.48422846, -1.53852738],\n",
       "       [-1.53852738, -2.54795941],\n",
       "       [-2.54795941, -3.28097239]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.c_[X_batch[0], y_batch[0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `OuputProjectionWrapper` 사용하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RNN 하나를 만들어 보겠습니다. 이 신경망은 100개의 순환 뉴런을 가지고 있고 각 훈련 샘플은 20개의 입력 길이로 구성되므로 20개의 타임 스텝에 펼칠 것입니다. 각 입력은 하나의 특성을 가집니다(각 시간에서의 값 하나). 타깃도 20개의 입력 시퀀스이고 하나의 값을 가집니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_steps = 20\n",
    "n_inputs = 1\n",
    "n_neurons = 100\n",
    "n_outputs = 1\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])\n",
    "\n",
    "cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n",
    "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "각 타임 스텝에서 크기가 100인 출력 벡터가 만들어 집니다. 하지만 각 타임 스텝에서 하나의 출력 값을 원합니다. 간단한 방법은 `OutputProjectionWrapper`로 셀을 감싸는 것입니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_steps = 20\n",
    "n_inputs = 1\n",
    "n_neurons = 100\n",
    "n_outputs = 1\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "cell = tf.contrib.rnn.OutputProjectionWrapper(\n",
    "    tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu),\n",
    "    output_size=n_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "\n",
    "loss = tf.reduce_mean(tf.square(outputs - y)) # MSE\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 \tMSE: 11.967253\n",
      "100 \tMSE: 0.5267912\n",
      "200 \tMSE: 0.1495199\n",
      "300 \tMSE: 0.07283236\n",
      "400 \tMSE: 0.06168884\n",
      "500 \tMSE: 0.05886846\n",
      "600 \tMSE: 0.05303419\n",
      "700 \tMSE: 0.048641633\n",
      "800 \tMSE: 0.04998502\n",
      "900 \tMSE: 0.047834214\n",
      "1000 \tMSE: 0.04796028\n",
      "1100 \tMSE: 0.04780812\n",
      "1200 \tMSE: 0.0415957\n",
      "1300 \tMSE: 0.047843512\n",
      "1400 \tMSE: 0.04253944\n"
     ]
    }
   ],
   "source": [
    "n_iterations = 1500\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        X_batch, y_batch = next_batch(batch_size, n_steps)\n",
    "        sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        if iteration % 100 == 0:\n",
    "            mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "            print(iteration, \"\\tMSE:\", mse)\n",
    "    \n",
    "    saver.save(sess, \"./my_time_series_model\") # not shown in the book"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_time_series_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:                          # 책에는 없음\n",
    "    saver.restore(sess, \"./my_time_series_model\")   # 책에는 없음\n",
    "\n",
    "    X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n",
    "    y_pred = sess.run(outputs, feed_dict={X: X_new})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-3.409996 ],\n",
       "        [-2.4543931],\n",
       "        [-1.1121919],\n",
       "        [ 0.7361519],\n",
       "        [ 2.16957  ],\n",
       "        [ 3.024978 ],\n",
       "        [ 3.4844422],\n",
       "        [ 3.366301 ],\n",
       "        [ 2.8970137],\n",
       "        [ 2.213668 ],\n",
       "        [ 1.6406606],\n",
       "        [ 1.5506338],\n",
       "        [ 1.9295691],\n",
       "        [ 2.7553375],\n",
       "        [ 3.8671415],\n",
       "        [ 5.0822144],\n",
       "        [ 6.0878325],\n",
       "        [ 6.6626954],\n",
       "        [ 6.645318 ],\n",
       "        [ 6.0674458]]], dtype=float32)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 16840 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"모델 테스트\", fontsize=14)\n",
    "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=12, label=\"샘플\")\n",
    "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markeredgewidth=0.5, markeredgecolor=\"b\", markersize=14, label=\"타깃\")\n",
    "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"예측\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"시간\")\n",
    "\n",
    "save_fig(\"time_series_pred_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `OutputProjectionWrapper` 사용하지 않기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_steps = 20\n",
    "n_inputs = 1\n",
    "n_neurons = 100\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "cell = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu)\n",
    "rnn_outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_outputs = 1\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n",
    "stacked_outputs = tf.layers.dense(stacked_rnn_outputs, n_outputs)\n",
    "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.square(outputs - y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 \tMSE: 13.907028\n",
      "100 \tMSE: 0.5062197\n",
      "200 \tMSE: 0.20180194\n",
      "300 \tMSE: 0.101789564\n",
      "400 \tMSE: 0.06869375\n",
      "500 \tMSE: 0.0650903\n",
      "600 \tMSE: 0.056519922\n",
      "700 \tMSE: 0.048897255\n",
      "800 \tMSE: 0.050609533\n",
      "900 \tMSE: 0.04794232\n",
      "1000 \tMSE: 0.047962066\n",
      "1100 \tMSE: 0.04990514\n",
      "1200 \tMSE: 0.041599542\n",
      "1300 \tMSE: 0.048676066\n",
      "1400 \tMSE: 0.043152742\n"
     ]
    }
   ],
   "source": [
    "n_iterations = 1500\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        X_batch, y_batch = next_batch(batch_size, n_steps)\n",
    "        sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        if iteration % 100 == 0:\n",
    "            mse = loss.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "            print(iteration, \"\\tMSE:\", mse)\n",
    "    \n",
    "    X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n",
    "    y_pred = sess.run(outputs, feed_dict={X: X_new})\n",
    "    \n",
    "    saver.save(sess, \"./my_time_series_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-3.433237  ],\n",
       "        [-2.454809  ],\n",
       "        [-1.0924242 ],\n",
       "        [ 0.70101255],\n",
       "        [ 2.1101534 ],\n",
       "        [ 3.0261526 ],\n",
       "        [ 3.4682128 ],\n",
       "        [ 3.321571  ],\n",
       "        [ 2.8254244 ],\n",
       "        [ 2.2395434 ],\n",
       "        [ 1.7126881 ],\n",
       "        [ 1.5265634 ],\n",
       "        [ 1.9084352 ],\n",
       "        [ 2.750864  ],\n",
       "        [ 3.8967836 ],\n",
       "        [ 5.1164637 ],\n",
       "        [ 6.1152263 ],\n",
       "        [ 6.6678505 ],\n",
       "        [ 6.6391854 ],\n",
       "        [ 6.045873  ]]], dtype=float32)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 14864 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"모델 테스트\", fontsize=14)\n",
    "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n",
    "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n",
    "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"시간\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 새로운 시퀀스 생성하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_time_series_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:                        # 책에는 없음\n",
    "    saver.restore(sess, \"./my_time_series_model\") # 책에는 없음\n",
    "\n",
    "    sequence = [0.] * n_steps\n",
    "    for iteration in range(300):\n",
    "        X_batch = np.array(sequence[-n_steps:]).reshape(1, n_steps, 1)\n",
    "        y_pred = sess.run(outputs, feed_dict={X: X_batch})\n",
    "        sequence.append(y_pred[0, -1, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 22308 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,4))\n",
    "plt.plot(np.arange(len(sequence)), sequence, \"b-\")\n",
    "plt.plot(t[:n_steps], sequence[:n_steps], \"b-\", linewidth=3)\n",
    "plt.xlabel(\"시간\")\n",
    "plt.ylabel(\"값\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_time_series_model\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 35224 characters]\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_time_series_model\")\n",
    "\n",
    "    sequence1 = [0. for i in range(n_steps)]\n",
    "    for iteration in range(len(t) - n_steps):\n",
    "        X_batch = np.array(sequence1[-n_steps:]).reshape(1, n_steps, 1)\n",
    "        y_pred = sess.run(outputs, feed_dict={X: X_batch})\n",
    "        sequence1.append(y_pred[0, -1, 0])\n",
    "\n",
    "    sequence2 = [time_series(i * resolution + t_min + (t_max-t_min/3)) for i in range(n_steps)]\n",
    "    for iteration in range(len(t) - n_steps):\n",
    "        X_batch = np.array(sequence2[-n_steps:]).reshape(1, n_steps, 1)\n",
    "        y_pred = sess.run(outputs, feed_dict={X: X_batch})\n",
    "        sequence2.append(y_pred[0, -1, 0])\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "plt.subplot(121)\n",
    "plt.plot(t, sequence1, \"b-\")\n",
    "plt.plot(t[:n_steps], sequence1[:n_steps], \"b-\", linewidth=3)\n",
    "plt.xlabel(\"시간\")\n",
    "plt.ylabel(\"값\", rotation=0)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(t, sequence2, \"b-\")\n",
    "plt.plot(t[:n_steps], sequence2[:n_steps], \"b-\", linewidth=3)\n",
    "plt.xlabel(\"시간\")\n",
    "save_fig(\"creative_sequence_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 심층 RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MultiRNNCell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 2\n",
    "n_steps = 5\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_neurons = 100\n",
    "n_layers = 3\n",
    "\n",
    "layers = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n",
    "          for layer in range(n_layers)]\n",
    "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(layers)\n",
    "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_batch = np.random.rand(2, n_steps, n_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    outputs_val, states_val = sess.run([outputs, states], feed_dict={X: X_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 5, 100)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs_val.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 여러 GPU에 심층 RNN 분산하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이렇게 사용해서는 **안됩니다**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(\"/gpu:0\"):  # 이 할당은 무시됩니다\n",
    "    layer1 = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n",
    "\n",
    "with tf.device(\"/gpu:1\"):  # 이 할당은 무시됩니다\n",
    "    layer2 = tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "대신 `DeviceCellWrapper`를 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "class DeviceCellWrapper(tf.contrib.rnn.RNNCell):\n",
    "  def __init__(self, device, cell):\n",
    "    self._cell = cell\n",
    "    self._device = device\n",
    "\n",
    "  @property\n",
    "  def state_size(self):\n",
    "    return self._cell.state_size\n",
    "\n",
    "  @property\n",
    "  def output_size(self):\n",
    "    return self._cell.output_size\n",
    "\n",
    "  def __call__(self, inputs, state, scope=None):\n",
    "    with tf.device(self._device):\n",
    "        return self._cell(inputs, state, scope)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 5\n",
    "n_steps = 20\n",
    "n_neurons = 100\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_steps, n_inputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "devices = [\"/cpu:0\", \"/cpu:0\", \"/cpu:0\"] # 만약 GPU가 세 개 있다면 [\"/gpu:0\", \"/gpu:1\", \"/gpu:2\"]로 바꿉니다\n",
    "cells = [DeviceCellWrapper(dev,tf.contrib.rnn.BasicRNNCell(num_units=n_neurons))\n",
    "         for dev in devices]\n",
    "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(cells)\n",
    "outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "또 다른 방법으로 텐서플로 1.1부터 `tf.contrib.rnn.DeviceWrapper` 클래스를 사용할 수 있습니다(텐서플로 1.2부터는 `tf.nn.rnn_cell.DeviceWrapper`가 되었습니다)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[-3.29020061e-02  1.17019236e-01 -4.58992608e-02 ...  3.84477153e-02\n",
      "   -5.24900258e-02  5.80436960e-02]\n",
      "  [ 3.54288332e-02 -7.42610693e-02 -6.42375052e-02 ... -2.45757148e-01\n",
      "   -1.77626804e-01 -3.26535292e-03]\n",
      "  [ 2.59726584e-01  1.44605249e-01  3.29312116e-01 ...  3.38821977e-01\n",
      "    5.98853417e-02 -8.31450596e-02]\n",
      "  ...\n",
      "  [ 1.18043743e-01 -2.41663873e-01 -8.27798247e-01 ... -1.85782462e-01\n",
      "    1.89682379e-01 -1.35261893e-01]\n",
      "  [ 2.87830859e-01  3.92953992e-01 -3.15236300e-01 ... -7.85605535e-02\n",
      "    4.32724983e-01 -6.61824420e-02]\n",
      "  [ 7.49779761e-01  6.11052871e-01 -3.01685542e-01 ...  1.95649698e-01\n",
      "   -1.62457466e-01 -5.23968339e-02]]\n",
      "\n",
      " [[-6.68027624e-02  7.35576972e-02 -2.90275402e-02 ... -7.79968277e-02\n",
      "   -3.65773253e-02 -4.66752201e-02]\n",
      "  [ 1.71546549e-01  4.28418722e-03  2.18812120e-03 ...  3.68538052e-02\n",
      "   -8.13241452e-02  9.01001915e-02]\n",
      "  [ 3.45635489e-02 -9.57525298e-02  1.59450233e-01 ...  1.63022086e-01\n",
      "   -6.83046281e-02  1.07990295e-01]\n",
      "  ...\n",
      "  [ 1.96414590e-01 -2.15230390e-01 -5.82537949e-01 ...  3.02532643e-01\n",
      "    5.47862291e-01 -1.06282756e-02]\n",
      "  [ 2.01206997e-01  2.52655987e-02 -5.02945222e-02 ...  9.56110135e-02\n",
      "   -2.03766495e-01 -1.52696133e-01]\n",
      "  [ 4.77758050e-01  1.66605622e-01 -1.56402126e-01 ...  1.67080894e-01\n",
      "   -2.03233853e-01  7.59940522e-05]]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    print(sess.run(outputs, feed_dict={X: np.random.rand(2, n_steps, n_inputs)}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 드롭아웃"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 1\n",
    "n_neurons = 100\n",
    "n_layers = 3\n",
    "n_steps = 20\n",
    "n_outputs = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "노트: `input_keep_prob` 매개변수는 플레이스홀더로 훈련하는 동안에는 어느 값이나 가능하고 테스트할 때는 1.0으로 지정합니다(드롭아웃을 끕니다)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "keep_prob = tf.placeholder_with_default(1.0, shape=())\n",
    "cells = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)\n",
    "         for layer in range(n_layers)]\n",
    "cells_drop = [tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=keep_prob)\n",
    "              for cell in cells]\n",
    "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(cells_drop)\n",
    "rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons])\n",
    "stacked_outputs = tf.layers.dense(stacked_rnn_outputs, n_outputs)\n",
    "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n",
    "\n",
    "loss = tf.reduce_mean(tf.square(outputs - y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 16.386942\n",
      "100 훈련 MSE: 4.25324\n",
      "200 훈련 MSE: 3.2349327\n",
      "300 훈련 MSE: 3.5502756\n",
      "400 훈련 MSE: 2.6301303\n",
      "500 훈련 MSE: 4.105153\n",
      "600 훈련 MSE: 3.5806417\n",
      "700 훈련 MSE: 3.3154318\n",
      "800 훈련 MSE: 3.5841472\n",
      "900 훈련 MSE: 3.4554102\n",
      "1000 훈련 MSE: 3.165893\n",
      "1100 훈련 MSE: 3.691963\n",
      "1200 훈련 MSE: 3.7539616\n",
      "1300 훈련 MSE: 2.4574566\n",
      "1400 훈련 MSE: 3.9499161\n"
     ]
    }
   ],
   "source": [
    "n_iterations = 1500\n",
    "batch_size = 50\n",
    "train_keep_prob = 0.5\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        X_batch, y_batch = next_batch(batch_size, n_steps)\n",
    "        _, mse = sess.run([training_op, loss],\n",
    "                          feed_dict={X: X_batch, y: y_batch,\n",
    "                                     keep_prob: train_keep_prob})\n",
    "        if iteration % 100 == 0:                   # not shown in the book\n",
    "            print(iteration, \"훈련 MSE:\", mse) # not shown\n",
    "    \n",
    "    saver.save(sess, \"./my_dropout_time_series_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_dropout_time_series_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_dropout_time_series_model\")\n",
    "\n",
    "    X_new = time_series(np.array(t_instance[:-1].reshape(-1, n_steps, n_inputs)))\n",
    "    y_pred = sess.run(outputs, feed_dict={X: X_new})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13676 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"모델 테스트\", fontsize=14)\n",
    "plt.plot(t_instance[:-1], time_series(t_instance[:-1]), \"bo\", markersize=10, label=\"instance\")\n",
    "plt.plot(t_instance[1:], time_series(t_instance[1:]), \"w*\", markersize=10, label=\"target\")\n",
    "plt.plot(t_instance[1:], y_pred[0,:,0], \"r.\", markersize=10, label=\"prediction\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"시간\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이런 드롭아웃이 이 경우엔 크게 도움이 안되네요. :/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "n_neurons = 150\n",
    "n_outputs = 10\n",
    "n_layers = 3\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "lstm_cells = [tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons)\n",
    "              for layer in range(n_layers)]\n",
    "multi_cell = tf.contrib.rnn.MultiRNNCell(lstm_cells)\n",
    "outputs, states = tf.nn.dynamic_rnn(multi_cell, X, dtype=tf.float32)\n",
    "top_layer_h_state = states[-1][1]\n",
    "logits = tf.layers.dense(top_layer_h_state, n_outputs, name=\"softmax\")\n",
    "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(?, 150) dtype=float32>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(?, 150) dtype=float32>),\n",
       " LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_5:0' shape=(?, 150) dtype=float32>, h=<tf.Tensor 'rnn/while/Exit_6:0' shape=(?, 150) dtype=float32>),\n",
       " LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_7:0' shape=(?, 150) dtype=float32>, h=<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 150) dtype=float32>))"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'rnn/while/Exit_8:0' shape=(?, 150) dtype=float32>"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_layer_h_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크 0 훈련 정확도 = 0.9533333 테스트 정확도 = 0.9549\n",
      "에포크 1 훈련 정확도 = 0.9866667 테스트 정확도 = 0.9733\n",
      "에포크 2 훈련 정확도 = 0.99333334 테스트 정확도 = 0.9769\n",
      "에포크 3 훈련 정확도 = 0.99333334 테스트 정확도 = 0.9807\n",
      "에포크 4 훈련 정확도 = 0.98 테스트 정확도 = 0.9845\n",
      "에포크 5 훈련 정확도 = 0.9866667 테스트 정확도 = 0.9815\n",
      "에포크 6 훈련 정확도 = 0.99333334 테스트 정확도 = 0.9845\n",
      "에포크 7 훈련 정확도 = 1.0 테스트 정확도 = 0.9876\n",
      "에포크 8 훈련 정확도 = 1.0 테스트 정확도 = 0.9877\n",
      "에포크 9 훈련 정확도 = 0.99333334 테스트 정확도 = 0.9881\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            X_batch = X_batch.reshape((batch_size, n_steps, n_inputs))\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(\"에포크\", epoch, \"훈련 정확도 =\", acc_train, \"테스트 정확도 =\", acc_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_cell = tf.contrib.rnn.LSTMCell(num_units=n_neurons, use_peepholes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_cell = tf.contrib.rnn.GRUCell(num_units=n_neurons)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 임베딩"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 섹션은 텐서플로의 [Word2Vec 튜토리얼](https://www.tensorflow.org/versions/r0.11/tutorials/word2vec/index.html)을 기반으로 합니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 데이터 추출"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from six.moves import urllib\n",
    "\n",
    "import errno\n",
    "import os\n",
    "import zipfile\n",
    "\n",
    "WORDS_PATH = \"datasets/words\"\n",
    "WORDS_URL = 'http://mattmahoney.net/dc/text8.zip'\n",
    "\n",
    "def mkdir_p(path):\n",
    "    \"\"\"디렉토리 생성, 이미 있다면 그냥 통과\n",
    "    \n",
    "    이 함수는 파이썬 2 버전을 지원하기 위해서입니다.\n",
    "    파이썬 3.2 이상이면 다음과 같이 쓸 수 있습니다:\n",
    "    >>> os.makedirs(path, exist_ok=True)\n",
    "    \"\"\"\n",
    "    try:\n",
    "        os.makedirs(path)\n",
    "    except OSError as exc:\n",
    "        if exc.errno == errno.EEXIST and os.path.isdir(path):\n",
    "            pass\n",
    "        else:\n",
    "            raise\n",
    "\n",
    "def fetch_words_data(words_url=WORDS_URL, words_path=WORDS_PATH):\n",
    "    os.makedirs(words_path, exist_ok=True)\n",
    "    zip_path = os.path.join(words_path, \"words.zip\")\n",
    "    if not os.path.exists(zip_path):\n",
    "        urllib.request.urlretrieve(words_url, zip_path)\n",
    "    with zipfile.ZipFile(zip_path) as f:\n",
    "        data = f.read(f.namelist()[0])\n",
    "    return data.decode(\"ascii\").split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = fetch_words_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['anarchism', 'originated', 'as', 'a', 'term']"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 사전 구축"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "vocabulary_size = 50000\n",
    "\n",
    "vocabulary = [(\"UNK\", None)] + Counter(words).most_common(vocabulary_size - 1)\n",
    "vocabulary = np.array([word for word, _ in vocabulary])\n",
    "dictionary = {word: code for code, word in enumerate(vocabulary)}\n",
    "data = np.array([dictionary.get(word, 0) for word in words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('anarchism originated as a term of abuse first used',\n",
       " array([5244, 3083,   12,    6,  195,    2, 3135,   46,   59]))"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\" \".join(words[:9]), data[:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'belonging didn as a term of extinct first used'"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\" \".join([vocabulary[word_index] for word_index in [5241, 3081, 12, 6, 195, 2, 3134, 46, 59]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('culottes', 0)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[24], data[24]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 배치 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from collections import deque\n",
    "\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "    global data_index\n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2 * skip_window\n",
    "    batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n",
    "    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n",
    "    span = 2 * skip_window + 1 # [ skip_window target skip_window ]\n",
    "    buffer = deque(maxlen=span)\n",
    "    for _ in range(span):\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    for i in range(batch_size // num_skips):\n",
    "        target = skip_window  # buffer 중간에 타깃 레이블을 둡니다\n",
    "        targets_to_avoid = [ skip_window ]\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid:\n",
    "                target = random.randint(0, span - 1)\n",
    "            targets_to_avoid.append(target)\n",
    "            batch[i * num_skips + j] = buffer[skip_window]\n",
    "            labels[i * num_skips + j, 0] = buffer[target]\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    return batch, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_index=0\n",
    "batch, labels = generate_batch(8, 2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3083, 3083,   12,   12,    6,    6,  195,  195], dtype=int32),\n",
       " ['originated', 'originated', 'as', 'as', 'a', 'a', 'term', 'term'])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch, [vocabulary[word] for word in batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[  12],\n",
       "        [5244],\n",
       "        [   6],\n",
       "        [3083],\n",
       "        [ 195],\n",
       "        [  12],\n",
       "        [   2],\n",
       "        [   6]], dtype=int32),\n",
       " ['as', 'anarchism', 'a', 'originated', 'term', 'as', 'of', 'a'])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels, [vocabulary[word] for word in labels[:, 0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모델 구성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "embedding_size = 128  # 임베딩 벡터 차원\n",
    "skip_window = 1       # 고려할 왼쪽과 오른쪽 단어의 개수\n",
    "num_skips = 2         # 레이블을 생성하기 위한 입력의 재사용 횟수\n",
    "\n",
    "# 가까운 이웃을 샘플링하기 위해 랜덤한 검증 세트를 만듭니다.\n",
    "# 검증 샘플은 가장 흔한 단어인 낮은 ID 번호를 가진 것으로 제한합니다.\n",
    "valid_size = 16     # 유사도를 평가하기 위해 랜덤하게 구성할 단어 세트 크기\n",
    "valid_window = 100  # 검증 샘플을 전체 샘플의 앞 부분에서만 선택합니다\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False)\n",
    "num_sampled = 64    # 부정 샘플링(negative sampling)의 수\n",
    "\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "# 입력 데이터\n",
    "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "valid_dataset = tf.constant(valid_examples, dtype=tf.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary_size = 50000\n",
    "embedding_size = 150\n",
    "\n",
    "# 입력을 위해 임베딩을 조회합니다\n",
    "init_embeds = tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)\n",
    "embeddings = tf.Variable(init_embeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_inputs = tf.placeholder(tf.int32, shape=[None])\n",
    "embed = tf.nn.embedding_lookup(embeddings, train_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-108-1dfd74cb904f>:18: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "# NCE 손실을 위한 변수를 만듭니다\n",
    "nce_weights = tf.Variable(\n",
    "    tf.truncated_normal([vocabulary_size, embedding_size],\n",
    "                        stddev=1.0 / np.sqrt(embedding_size)))\n",
    "nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "# 배치에서 NCE 손실의 평균을 계산합니다.Compute the average NCE loss for the batch.\n",
    "# tf.nce_loss는 자동으로 손실을 평가할 때마다 음성 레이블에서 새로운 샘플을 뽑습니다.\n",
    "loss = tf.reduce_mean(\n",
    "    tf.nn.nce_loss(nce_weights, nce_biases, train_labels, embed,\n",
    "                   num_sampled, vocabulary_size))\n",
    "\n",
    "# Adam 옵티마이저\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "# 미니배치 샘플과 모든 임베딩 사이의 코사인 유사도를 계산합니다\n",
    "norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), axis=1, keep_dims=True))\n",
    "normalized_embeddings = embeddings / norm\n",
    "valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)\n",
    "similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "\n",
    "# 초기화를 위한 연산\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모델 훈련"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "반복: 0\t스텝  0 에서의 평균 손실:  294.72406005859375\n",
      "would에 가장 가까운 단어: lindy, humorously, engagements, bek, phoenicia, showcase, losing, nf,\n",
      "on에 가장 가까운 단어: imprints, arbitrarily, cis, prefect, saccharomyces, jj, classmate, ave,\n",
      "four에 가장 가까운 단어: citations, insist, tank, macarthur, uneducated, plissken, riordan, vasili,\n",
      "his에 가장 가까운 단어: generous, archie, themes, achieved, somehow, bartle, value, lexis,\n",
      "often에 가장 가까운 단어: deception, sticks, hitomaro, subtract, calculates, darrow, lavender, anthony,\n",
      "in에 가장 가까운 단어: termini, bubonic, elaborating, commence, tobacco, subdivided, competing, puppets,\n",
      "an에 가장 가까운 단어: queene, lavon, chestnuts, solomon, clayton, voce, abolishes, patched,\n",
      "eight에 가장 가까운 단어: rig, retaliated, spr, bump, poinsot, wild, sportspeople, bison,\n",
      "these에 가장 가까운 단어: conferences, ypres, neck, refrain, bessarabia, cord, gravely, reconciling,\n",
      "nine에 가장 가까운 단어: pittsburg, trivia, carol, menorah, overwriting, audition, develop, ranks,\n",
      "called에 가장 가까운 단어: bon, dealing, stallion, hata, ise, vadim, sizeable, forgetfulness,\n",
      "about에 가장 가까운 단어: euphoria, hinayana, vikernes, centralization, lt, washing, promontory, mop,\n",
      "up에 가장 가까운 단어: abet, drives, addictions, piercing, constants, quipped, misr, rainy,\n",
      "one에 가장 가까운 단어: syntactically, broughton, kuru, radiodiffusion, chute, alan, announcing, pashtuns,\n",
      "and에 가장 가까운 단어: soft, visas, fasts, lillard, drowsiness, coffin, komondor, vulnerability,\n",
      "been에 가장 가까운 단어: sang, expositor, reloading, otis, linear, widen, ephraem, astros,\n",
      "반복: 2000\t스텝  2000 에서의 평균 손실:  131.0253740901947\n",
      "반복: 4000\t스텝  4000 에서의 평균 손실:  62.29951333236694\n",
      "반복: 6000\t스텝  6000 에서의 평균 손실:  41.85774783444405\n",
      "반복: 8000\t스텝  8000 에서의 평균 손실:  30.861831660270692\n",
      "반복: 10000\t스텝  10000 에서의 평균 손실:  26.37321024131775\n",
      "would에 가장 가까운 단어: can, mk, to, mari, may, should, doubleday, outdoors,\n",
      "on에 가장 가까운 단어: cin, bambaataa, and, of, in, halfback, cca, dh,\n",
      "four에 가장 가까운 단어: three, five, one, six, seven, two, nine, zero,\n",
      "his에 가장 가까운 단어: the, propositional, achill, livelihood, merchandising, intercollegiate, bukhara, bicycle,\n",
      "often에 가장 가까운 단어: dubbing, commonly, utf, altaic, also, signed, and, which,\n",
      "in에 가장 가까운 단어: and, of, from, trench, for, the, altaic, coulomb,\n",
      "an에 가장 가까운 단어: nen, anointed, rat, chi, antigua, abandonment, abscess, barbara,\n",
      "eight에 가장 가까운 단어: one, six, nine, seven, five, three, zero, four,\n",
      "these에 가장 가까운 단어: bremer, notice, bambaataa, nage, automorphism, olympias, psalms, mannerisms,\n",
      "nine에 가장 가까운 단어: one, six, eight, zero, seven, five, four, two,\n",
      "called에 가장 가까운 단어: also, is, loser, cma, amalgamated, hat, difficulty, viva,\n",
      "about에 가장 가까운 단어: altaic, donation, selenium, numidia, preston, paramount, seven, bragg,\n",
      "up에 가장 가까운 단어: whittier, maguey, windmill, bartholomew, dry, qualities, aardwolf, coffee,\n",
      "one에 가장 가까운 단어: nine, eight, six, seven, four, two, three, zero,\n",
      "and에 가장 가까운 단어: in, the, UNK, of, altaic, one, for, a,\n",
      "been에 가장 가까운 단어: have, cma, enver, was, had, diodorus, pseudonym, argued,\n"
     ]
    }
   ],
   "source": [
    "num_steps = 10001\n",
    "\n",
    "with tf.Session() as session:\n",
    "    init.run()\n",
    "\n",
    "    average_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        print(\"\\r반복: {}\".format(step), end=\"\\t\")\n",
    "        batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)\n",
    "        feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}\n",
    "\n",
    "        # 훈련 연산을 평가하여 스텝을 한 단계를 업데이트합니다(session.run()에서 반환된 값을 사용합니다)\n",
    "        _, loss_val = session.run([training_op, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "            # 평균 손실은 2000개 배치에 대한 손실의 추정입니다.\n",
    "            print(\"스텝 \", step, \"에서의 평균 손실: \", average_loss)\n",
    "            average_loss = 0\n",
    "\n",
    "        # 이 코드는 비용이 많이 듭니다 (500 스텝마다 ~20%씩 느려집니다)\n",
    "        if step % 10000 == 0:\n",
    "            sim = similarity.eval()\n",
    "            for i in range(valid_size):\n",
    "                valid_word = vocabulary[valid_examples[i]]\n",
    "                top_k = 8 # 가장 가까운 단어의 개수\n",
    "                nearest = (-sim[i, :]).argsort()[1:top_k+1]\n",
    "                log_str = \"%s에 가장 가까운 단어:\" % valid_word\n",
    "                for k in range(top_k):\n",
    "                    close_word = vocabulary[nearest[k]]\n",
    "                    log_str = \"%s %s,\" % (log_str, close_word)\n",
    "                print(log_str)\n",
    "\n",
    "    final_embeddings = normalized_embeddings.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "마지막 임베딩을 저장합니다(물론 텐서플로의 `Saver`를 사용해도 됩니다):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"./my_final_embeddings.npy\", final_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 임베딩 그래프"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_with_labels(low_dim_embs, labels):\n",
    "    assert low_dim_embs.shape[0] >= len(labels), \"임베딩보다 레이블이 많습니다.\"\n",
    "    plt.figure(figsize=(18, 18))  # 인치 크기\n",
    "    for i, label in enumerate(labels):\n",
    "        x, y = low_dim_embs[i,:]\n",
    "        plt.scatter(x, y)\n",
    "        plt.annotate(label,\n",
    "                     xy=(x, y),\n",
    "                     xytext=(5, 2),\n",
    "                     textcoords='offset points',\n",
    "                     ha='right',\n",
    "                     va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 305616 characters]\n",
      "text/plain": [
       "<Figure size 1296x1296 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "plot_only = 500\n",
    "low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only,:])\n",
    "labels = [vocabulary[i] for i in range(plot_only)]\n",
    "plot_with_labels(low_dim_embs, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 기계 번역"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`basic_rnn_seq2seq()` 함수는 간단한 인코더/디코더 모델을 만듭니다. 먼저 `encoder_inputs`를 상태 벡터로 인코딩하는 RNN을 실행하고 그다음 `decoder_inputs`을 마지막 인코더 상태로 초기화시킨 디코더를 실행합니다. 인코더와 디코더는 같은 RNN 셀 타입을 사용하지만 파라미터를 공유하지는 않습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "reset_graph()\n",
    "\n",
    "n_steps = 50\n",
    "n_neurons = 200\n",
    "n_layers = 3\n",
    "num_encoder_symbols = 20000\n",
    "num_decoder_symbols = 20000\n",
    "embedding_size = 150\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.int32, [None, n_steps]) # 영어 문장\n",
    "Y = tf.placeholder(tf.int32, [None, n_steps]) # 프랑스어 번역\n",
    "W = tf.placeholder(tf.float32, [None, n_steps - 1, 1])\n",
    "Y_input = Y[:, :-1]\n",
    "Y_target = Y[:, 1:]\n",
    "\n",
    "encoder_inputs = tf.unstack(tf.transpose(X)) # 1D 텐서의 리스트\n",
    "decoder_inputs = tf.unstack(tf.transpose(Y_input)) # 1D 텐서의 리스트\n",
    "\n",
    "lstm_cells = [tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons)\n",
    "              for layer in range(n_layers)]\n",
    "cell = tf.contrib.rnn.MultiRNNCell(lstm_cells)\n",
    "\n",
    "output_seqs, states = tf.contrib.legacy_seq2seq.embedding_rnn_seq2seq(\n",
    "    encoder_inputs,\n",
    "    decoder_inputs,\n",
    "    cell,\n",
    "    num_encoder_symbols,\n",
    "    num_decoder_symbols,\n",
    "    embedding_size)\n",
    "\n",
    "logits = tf.transpose(tf.unstack(output_seqs), perm=[1, 0, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits_flat = tf.reshape(logits, [-1, num_decoder_symbols])\n",
    "Y_target_flat = tf.reshape(Y_target, [-1])\n",
    "W_flat = tf.reshape(W, [-1])\n",
    "xentropy = W_flat * tf.nn.sparse_softmax_cross_entropy_with_logits(labels=Y_target_flat, logits=logits_flat)\n",
    "loss = tf.reduce_mean(xentropy)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 해답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "부록 A 참조."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. 임베딩된 레버(Reber) 문법"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저 문법에 맞는 문자열을 생성하는 함수가 필요합니다. 이 문법은 각 상태에서 가능한 전이 상태의 리스트입니다. 하나의 전이는 출력할 문자열(또는 생성할 문법)과 다음 상태를 지정합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import choice, seed\n",
    "\n",
    "# 일관된 출력을 위한 유사난수 초기화\n",
    "seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "default_reber_grammar = [\n",
    "    [(\"B\", 1)],           # (상태 0) =B=>(상태 1)\n",
    "    [(\"T\", 2), (\"P\", 3)], # (상태 1) =T=>(상태 2) or =P=>(상태 3)\n",
    "    [(\"S\", 2), (\"X\", 4)], # (상태 2) =S=>(상태 2) or =X=>(상태 4)\n",
    "    [(\"T\", 3), (\"V\", 5)], # 등등..\n",
    "    [(\"X\", 3), (\"S\", 6)],\n",
    "    [(\"P\", 4), (\"V\", 6)],\n",
    "    [(\"E\", None)]]        # (상태 6) =E=>(종료 상태)\n",
    "\n",
    "embedded_reber_grammar = [\n",
    "    [(\"B\", 1)],\n",
    "    [(\"T\", 2), (\"P\", 3)],\n",
    "    [(default_reber_grammar, 4)],\n",
    "    [(default_reber_grammar, 5)],\n",
    "    [(\"T\", 6)],\n",
    "    [(\"P\", 6)],\n",
    "    [(\"E\", None)]]\n",
    "\n",
    "def generate_string(grammar):\n",
    "    state = 0\n",
    "    output = []\n",
    "    while state is not None:\n",
    "        production, state = choice(grammar[state])\n",
    "        if isinstance(production, list):\n",
    "            production = generate_string(grammar=production)\n",
    "        output.append(production)\n",
    "    return \"\".join(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "기본 레버 문법에 맞는 문자열을 몇 개 만들어 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BTXXTTTTVPXTTTTTVPSE BTXSE BTXXTVPSE BTXXVPSE BTSSXXTTVVE BTXSE BTSSSXSE BPTTTVVE BTXXVVE BPTTVVE BTSXXTTTTVPSE BPTTVVE BPTVPSE BPTTVPXVVE BPVPXTTTVPXTVPSE BTXSE BPTTTTVPXTTTTTTTVPXVVE BPTVVE BTXSE BPTTTVVE BTSXXVPSE BTXXTTTTTVVE BPTTVPSE BPVVE BPTTTVPXVPXTTTTTVPXTTVVE "
     ]
    }
   ],
   "source": [
    "for _ in range(25):\n",
    "    print(generate_string(default_reber_grammar), end=\" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋습니다. 이제 임베딩된 레버 문법에 맞는 문자열을 몇 개 만들어 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPBPTVVEPE BTBPTVPXVVETE BPBPTTTVVEPE BPBTXSEPE BPBPTTTTTVPSEPE BTBTSXSETE BPBPVPSEPE BPBPVVEPE BPBTXSEPE BPBTSXSEPE BTBPTTVVETE BPBPVVEPE BTBTXSETE BPBPTTVVEPE BTBTSXXVVETE BTBTXXTVPXTVPSETE BTBPTVVETE BPBPVPXTTVPXTVVEPE BTBTXSETE BPBTXSEPE BPBTSXXTVPSEPE BPBPVVEPE BPBPTTTTTTTTTTVPXVVEPE BPBPVVEPE BPBPVVEPE "
     ]
    }
   ],
   "source": [
    "for _ in range(25):\n",
    "    print(generate_string(embedded_reber_grammar), end=\" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋네요, 이제 이 문법을 따르지 않는 문자열을 생성할 함수를 만듭니다. 무작위하게 문자열을 만들 수 있지만 그렇게 하면 너무 문제가 쉬워지므로 대신 문법을 따르는 문자열을 만든 후 하나의 문자만 바꾸어 놓도록 하겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_corrupted_string(grammar, chars=\"BEPSTVX\"):\n",
    "    good_string = generate_string(grammar)\n",
    "    index = np.random.randint(len(good_string))\n",
    "    good_char = good_string[index]\n",
    "    bad_char = choice(list(set(chars) - set(good_char)))\n",
    "    return good_string[:index] + bad_char + good_string[index + 1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "잘못된 문자열 몇 개를 만들어 보죠:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPBPVPPEPE BPBSXSEPE BPBPTVVTPE BTBPEPSETE BTBPVVVTE BPBTSSXXTSTVVEPE BPVTSXXTVPSEPE BPBTXSPPE BTBPTTTVPSTTE BPBTSXXTTTXTTVVEPE BPBBXXVPXTVPXTTVVEPE BPBPTTVXEPE BPBPVVEEE BPPPTTVVEPE BPBPVESEPE BPBTBXXVVEPE BVBPTTTVPXVVETE BPBTSSXVEPE BPBPVXEPE BVBTXSEPE BTBPTVPXVPXVVETV EPBTSXXTVPXVPSEPE BPBTSXXTTTVBSEPE BPBPVPXVVPPE BTBTSBSETE "
     ]
    }
   ],
   "source": [
    "for _ in range(25):\n",
    "    print(generate_corrupted_string(embedded_reber_grammar), end=\" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "문자열을 바로 RNN에 주입할 수는 없습니다. 먼저 벡터의 연속으로 바꾸어야 합니다. 각 벡터는 원-핫 인코딩을 사용하여 하나의 문자를 나타냅니다. 예를 들어, 벡터 `[1, 0, 0, 0, 0, 0, 0]`는 문자 \"B\"를 나타내고 벡터 `[0, 1, 0, 0, 0, 0, 0]`는 문자 \"E\"를 나타내는 식입니다. 이런 원-핫 벡터의 연속으로 문자열을 바꾸는 함수를 작성해 보겠습니다. 문자열이 `n_steps`보다 짧으면 0 벡터로 패딩됩니다(나중에, 텐서플로에게 각 문자열의 실제 길이를 `sequence_length` 매개변수로 전달할 것입니다)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "def string_to_one_hot_vectors(string, n_steps, chars=\"BEPSTVX\"):\n",
    "    char_to_index = {char: index for index, char in enumerate(chars)}\n",
    "    output = np.zeros((n_steps, len(chars)), dtype=np.int32)\n",
    "    for index, char in enumerate(string):\n",
    "        output[index, char_to_index[char]] = 1.\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [1, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 1, 0, 0, 0],\n",
       "       [0, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_to_one_hot_vectors(\"BTBTXSETE\", 12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 50%는 올바른 문자열 50%는 잘못된 문자열로 이루어진 데이터셋을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_dataset(size):\n",
    "    good_strings = [generate_string(embedded_reber_grammar)\n",
    "                    for _ in range(size // 2)]\n",
    "    bad_strings = [generate_corrupted_string(embedded_reber_grammar)\n",
    "                   for _ in range(size - size // 2)]\n",
    "    all_strings = good_strings + bad_strings\n",
    "    n_steps = max([len(string) for string in all_strings])\n",
    "    X = np.array([string_to_one_hot_vectors(string, n_steps)\n",
    "                  for string in all_strings])\n",
    "    seq_length = np.array([len(string) for string in all_strings])\n",
    "    y = np.array([[1] for _ in range(len(good_strings))] +\n",
    "                 [[0] for _ in range(len(bad_strings))])\n",
    "    rnd_idx = np.random.permutation(size)\n",
    "    return X[rnd_idx], seq_length[rnd_idx], y[rnd_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, l_train, y_train = generate_dataset(10000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "첫 번째 훈련 샘플을 확인해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [1, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 1, 0],\n",
       "       [0, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 0, 1, 0],\n",
       "       [0, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 0, 1, 0],\n",
       "       [0, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 0, 1, 0],\n",
       "       [0, 0, 0, 0, 0, 1, 0],\n",
       "       [0, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 0],\n",
       "       [0, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터셋에서 가장 긴 문자열 때문에 패딩된 0 벡터가 많습니다. 문자열 길이가 얼마나 될까요?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "타깃 클래스는?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0])"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "아주 좋습니다! 올바른 문자열을 구분할 RNN을 만들 준비가 되었습니다. 앞서 MNIST 이미지를 분류하기 위해 만든 것과 매우 비슷한 시퀀스 분류기를 만듭니다. 차이점은 다음 두 가지입니다:\n",
    "* 첫째, 입력 문자열이 가변 길이이므로 `dynamic_rnn()` 함수를 호출할 때 `sequence_length`를 지정해야 합니다.\n",
    "* 둘째, 이진 분류기이므로 출력 뉴런은 하나만 필요합니다. 이 뉴런은 각 문자열에 대해 올바른 문자열일 추정 로그 확률을 출력할 것입니다. 다중 클래스 분류에서는 `sparse_softmax_cross_entropy_with_logits()`를 사용했지만 이진 분류에서는 `sigmoid_cross_entropy_with_logits()`를 사용합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "possible_chars = \"BEPSTVX\"\n",
    "n_inputs = len(possible_chars)\n",
    "n_neurons = 30\n",
    "n_outputs = 1\n",
    "\n",
    "learning_rate = 0.02\n",
    "momentum = 0.95\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, None, n_inputs], name=\"X\")\n",
    "seq_length = tf.placeholder(tf.int32, [None], name=\"seq_length\")\n",
    "y = tf.placeholder(tf.float32, [None, 1], name=\"y\")\n",
    "\n",
    "gru_cell = tf.contrib.rnn.GRUCell(num_units=n_neurons)\n",
    "outputs, states = tf.nn.dynamic_rnn(gru_cell, X, dtype=tf.float32,\n",
    "                                    sequence_length=seq_length)\n",
    "\n",
    "logits = tf.layers.dense(states, n_outputs, name=\"logits\")\n",
    "y_pred = tf.cast(tf.greater(logits, 0.), tf.float32, name=\"y_pred\")\n",
    "y_proba = tf.nn.sigmoid(logits, name=\"y_proba\")\n",
    "\n",
    "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n",
    "                                       momentum=momentum,\n",
    "                                       use_nesterov=True)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.equal(y_pred, y, name=\"correct\")\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훈련하는 동안 진척 상황을 확인할 수 있도록 검증 세트를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val, l_val, y_val = generate_dataset(5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   0  훈련 손실: 0.6829, 정확도: 54.00%  검증 세트 정확도: 58.80%\n",
      "   1  훈련 손실: 0.6518, 정확도: 64.00%  검증 세트 정확도: 62.38%\n",
      "   2  훈련 손실: 0.6082, 정확도: 72.00%  검증 세트 정확도: 67.86%\n",
      "   3  훈련 손실: 0.6268, 정확도: 64.00%  검증 세트 정확도: 66.56%\n",
      "   4  훈련 손실: 0.4726, 정확도: 74.00%  검증 세트 정확도: 77.60%\n",
      "   5  훈련 손실: 0.3666, 정확도: 84.00%  검증 세트 정확도: 84.02%\n",
      "   6  훈련 손실: 0.3469, 정확도: 90.00%  검증 세트 정확도: 81.58%\n",
      "   7  훈련 손실: 0.2350, 정확도: 76.00%  검증 세트 정확도: 71.58%\n",
      "   8  훈련 손실: 0.2000, 정확도: 94.00%  검증 세트 정확도: 95.86%\n",
      "   9  훈련 손실: 0.4007, 정확도: 74.00%  검증 세트 정확도: 76.34%\n",
      "  10  훈련 손실: 0.3353, 정확도: 88.00%  검증 세트 정확도: 90.50%\n",
      "  11  훈련 손실: 0.0116, 정확도: 100.00%  검증 세트 정확도: 99.44%\n",
      "  12  훈련 손실: 0.0029, 정확도: 100.00%  검증 세트 정확도: 99.90%\n",
      "  13  훈련 손실: 0.0016, 정확도: 100.00%  검증 세트 정확도: 99.96%\n",
      "  14  훈련 손실: 0.0011, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  15  훈련 손실: 0.0010, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  16  훈련 손실: 0.0009, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  17  훈련 손실: 0.0007, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  18  훈련 손실: 0.0006, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  19  훈련 손실: 0.0006, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  20  훈련 손실: 0.0005, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  21  훈련 손실: 0.0004, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  22  훈련 손실: 0.0004, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  23  훈련 손실: 0.0004, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  24  훈련 손실: 0.0003, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  25  훈련 손실: 0.0003, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  26  훈련 손실: 0.0003, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  27  훈련 손실: 0.0003, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  28  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  29  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  30  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  31  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  32  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  33  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  34  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  35  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  36  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  37  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  38  훈련 손실: 0.0002, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  39  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  40  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  41  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  42  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  43  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  44  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  45  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  46  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  47  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  48  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n",
      "  49  훈련 손실: 0.0001, 정확도: 100.00%  검증 세트 정확도: 100.00%\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        X_batches = np.array_split(X_train, len(X_train) // batch_size)\n",
    "        l_batches = np.array_split(l_train, len(l_train) // batch_size)\n",
    "        y_batches = np.array_split(y_train, len(y_train) // batch_size)\n",
    "        for X_batch, l_batch, y_batch in zip(X_batches, l_batches, y_batches):\n",
    "            loss_val, _ = sess.run(\n",
    "                [loss, training_op],\n",
    "                feed_dict={X: X_batch, seq_length: l_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, seq_length: l_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: X_val, seq_length: l_val, y: y_val})\n",
    "        print(\"{:4d}  훈련 손실: {:.4f}, 정확도: {:.2f}%  검증 세트 정확도: {:.2f}%\".format(\n",
    "            epoch, loss_val, 100 * acc_train, 100 * acc_val))\n",
    "        saver.save(sess, \"./my_reber_classifier\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 두 개의 문자열에 이 RNN을 테스트해 보죠. 첫 번째는 잘못된 것이고 두 번째는 올바른 것입니다. 이 문자열은 마지막에서 두 번째 글자만 다릅니다. RNN이 이를 맞춘다면 두 번째 문자가 항상 끝에서 두 번째 문자와 같아야 한다는 패턴을 알게 됐다는 것을 의미합니다. 이렇게 하려면 꽤 긴 단기 기억(long short-term memory)이 필요합니다(그래서 GRU 셀을 사용했습니다)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from my_reber_classifier\n",
      "\n",
      "레버 문자열일 추정 확률:\n",
      "BPBTSSSSSSSSSSSSXXTTTTTVPXTTVPXTTTTTTTVPXVPXVPXTTTVVETE: 0.00%\n",
      "BPBTSSSSSSSSSSSSXXTTTTTVPXTTVPXTTTTTTTVPXVPXVPXTTTVVEPE: 1.00%\n"
     ]
    }
   ],
   "source": [
    "test_strings = [\n",
    "    \"BPBTSSSSSSSSSSSSXXTTTTTVPXTTVPXTTTTTTTVPXVPXVPXTTTVVETE\",\n",
    "    \"BPBTSSSSSSSSSSSSXXTTTTTVPXTTVPXTTTTTTTVPXVPXVPXTTTVVEPE\"]\n",
    "l_test = np.array([len(s) for s in test_strings])\n",
    "max_length = l_test.max()\n",
    "X_test = [string_to_one_hot_vectors(s, n_steps=max_length)\n",
    "          for s in test_strings]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"my_reber_classifier\")\n",
    "    y_proba_val = y_proba.eval(feed_dict={X: X_test, seq_length: l_test})\n",
    "\n",
    "print()\n",
    "print(\"레버 문자열일 추정 확률:\")\n",
    "for index, string in enumerate(test_strings):\n",
    "    print(\"{}: {:.2f}%\".format(string, y_proba_val[index][0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "쨘! 잘 작동하네요. 이 RNN이 완벽한 신뢰도로 정확한 답을 냈습니다. :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. 과 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coming soon..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
