{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.5.5\n",
      "IPython 6.3.0\n",
      "\n",
      "numpy 1.14.2\n",
      "sklearn 0.19.1\n",
      "scipy 1.0.1\n",
      "matplotlib 2.2.2\n",
      "tensorflow 1.7.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,sklearn,scipy,matplotlib,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**15장 – 오토인코더**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 15장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"autoencoders\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "28x28 흑백 이미지를 그리기 위한 유틸리티 함수:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_multiple_images(images, n_rows, n_cols, pad=2):\n",
    "    images = images - images.min()  # 최소값을 0으로 만들어 패딩이 하얗게 보이도록 합니다.\n",
    "    w,h = images.shape[1:]\n",
    "    image = np.zeros(((w+pad)*n_rows+pad, (h+pad)*n_cols+pad))\n",
    "    for y in range(n_rows):\n",
    "        for x in range(n_cols):\n",
    "            image[(y*(h+pad)+pad):(y*(h+pad)+pad+h),(x*(w+pad)+pad):(x*(w+pad)+pad+w)] = images[y*n_cols+x]\n",
    "    plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 선형 오토인코더를 사용한 PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3D 데이터셋을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy.random as rnd\n",
    "\n",
    "rnd.seed(4)\n",
    "m = 200\n",
    "w1, w2 = 0.1, 0.3\n",
    "noise = 0.1\n",
    "\n",
    "angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5\n",
    "data = np.empty((m, 3))\n",
    "data[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2\n",
    "data[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2\n",
    "data[:, 2] = data[:, 0] * w1 + data[:, 1] * w2 + noise * rnd.randn(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터를 정규화합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(data[:100])\n",
    "X_test = scaler.transform(data[100:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "오토인코더를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "reset_graph()\n",
    "\n",
    "n_inputs = 3\n",
    "n_hidden = 2  # 코딩 유닛\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden = tf.layers.dense(X, n_hidden)\n",
    "outputs = tf.layers.dense(hidden, n_outputs)\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(reconstruction_loss)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 1000\n",
    "codings = hidden\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        training_op.run(feed_dict={X: X_train})\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 6580 characters]\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(4,3))\n",
    "plt.plot(codings_val[:,0], codings_val[:, 1], \"b.\")\n",
    "plt.xlabel(\"$z_1$\", fontsize=18)\n",
    "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n",
    "save_fig(\"linear_autoencoder_pca_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 적층 오토인코더"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST 데이터셋을 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "tf.logging.set_verbosity(tf.logging.ERROR)  # deprecated 경고 메세지를 출력하지 않기 위해 \n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 한 번에 모든 층을 훈련하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3개의 은닉층과 1개의 출력층(즉, 두 개를 적층)을 가진 적층 오토인코더를 만들어 보겠습니다. ELU 활성화 함수와 He 초기화, L2 정규화를 사용하겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "he_init = tf.contrib.layers.variance_scaling_initializer() # He 초기화\n",
    "#아래와 동일합니다:\n",
    "#he_init = lambda shape, dtype=tf.float32: tf.truncated_normal(shape, 0., stddev=np.sqrt(2/shape[0]))\n",
    "l2_regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "my_dense_layer = partial(tf.layers.dense,\n",
    "                         activation=tf.nn.elu,\n",
    "                         kernel_initializer=he_init,\n",
    "                         kernel_regularizer=l2_regularizer)\n",
    "\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3 = my_dense_layer(hidden2, n_hidden3)\n",
    "outputs = my_dense_layer(hidden3, n_outputs, activation=None)\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "loss = tf.add_n([reconstruction_loss] + reg_losses)\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver() # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 훈련시켜 보죠! 여기에서는 타깃 값을 주입하지 않습니다(`y_batch`가 사용되지 않습니다). 이는 비지도 학습입니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.022107124\n",
      "1 훈련 MSE: 0.012154639\n",
      "2 훈련 MSE: 0.010653543\n",
      "3 훈련 MSE: 0.010217447\n",
      "4 훈련 MSE: 0.010719749\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\") # 책에는 없음\n",
    "            sys.stdout.flush()                                          # 책에는 없음\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})   # 책에는 없음\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)           # 책에는 없음\n",
    "        saver.save(sess, \"./my_model_all_layers.ckpt\")                  # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 함수는 모델을 로드하고 테스트 세트에서 이를 평가합니다(재구성 오차를 측정합니다). 그런 다음 원본 이미지와 재구성 이미지를 그립니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):\n",
    "    with tf.Session() as sess:\n",
    "        if model_path:\n",
    "            saver.restore(sess, model_path)\n",
    "        X_test = mnist.test.images[:n_test_digits]\n",
    "        outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "    fig = plt.figure(figsize=(8, 3 * n_test_digits))\n",
    "    for digit_index in range(n_test_digits):\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "        plot_image(X_test[digit_index])\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "        plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_all_layers.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 10036 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_all_layers.ckpt\")\n",
    "save_fig(\"reconstruction_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 가중치 묶기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "인코더와 디코더의 가중치를 묶는 일은 자주 있습니다(`weights_decoder = tf.transpose(weights_encoder)`). 안타깝지만 `tf.layers.dense()` 함수를 사용해서 이렇게 하기는 불가능합니다(또는 매우 어렵습니다). 수동으로 직접 오토인코더를 만들어야 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # 가중치 묶기\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # 가중치 묶기\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = reconstruction_loss + reg_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.01506695\n",
      "1 훈련 MSE: 0.016488748\n",
      "2 훈련 MSE: 0.017375939\n",
      "3 훈련 MSE: 0.016878333\n",
      "4 훈련 MSE: 0.015587721\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        saver.save(sess, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_tying_weights.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 9596 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 여러 개의 그래프에서 오토인토더를 따로따로 훈련하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하나의 오토인코더를 따로따로 훈련하는 방법이 많이 있습니다. 첫 번째 방법은 각 오토인코더를 다른 그래프를 사용하여 훈련하는 것입니다. 그런 다음 이런 오토인코더의 가중치와 편향을 복사해 초깃값으로 지정해서 적층 오토인코더를 만듭니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하나의 오토인코더를 훈련하고 변환된 훈련 세트(즉, 은닉층의 출력)와 모델 파라미터를 반환하는 함수를 만들겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "def train_autoencoder(X_train, n_neurons, n_epochs, batch_size,\n",
    "                      learning_rate = 0.01, l2_reg = 0.0005, seed=42,\n",
    "                      hidden_activation=tf.nn.elu,\n",
    "                      output_activation=tf.nn.elu):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        tf.set_random_seed(seed)\n",
    "\n",
    "        n_inputs = X_train.shape[1]\n",
    "\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "        \n",
    "        my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(l2_reg))\n",
    "\n",
    "        hidden = my_dense_layer(X, n_neurons, activation=hidden_activation, name=\"hidden\")\n",
    "        outputs = my_dense_layer(hidden, n_inputs, activation=output_activation, name=\"outputs\")\n",
    "\n",
    "        reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "        reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "        loss = tf.add_n([reconstruction_loss] + reg_losses)\n",
    "\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "        init = tf.global_variables_initializer()\n",
    "\n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        init.run()\n",
    "        for epoch in range(n_epochs):\n",
    "            n_batches = len(X_train) // batch_size\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                indices = rnd.permutation(len(X_train))[:batch_size]\n",
    "                X_batch = X_train[indices]\n",
    "                sess.run(training_op, feed_dict={X: X_batch})\n",
    "            loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n",
    "        hidden_val = hidden.eval(feed_dict={X: X_train})\n",
    "        return hidden_val, params[\"hidden/kernel:0\"], params[\"hidden/bias:0\"], params[\"outputs/kernel:0\"], params[\"outputs/bias:0\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 두 개의 오토인코더를 훈련시켜 보죠. 첫 번째는 훈련 데이터를 사용하고 두 번째는 첫 번째 오토인코더의 은닉층 출력을 사용해 훈련시킵니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.018564083\n",
      "1 훈련 MSE: 0.018759882\n",
      "2 훈련 MSE: 0.018457456\n",
      "3 훈련 MSE: 0.019239973\n",
      "0 훈련 MSE: 0.0042280294\n",
      "1 훈련 MSE: 0.004805386\n",
      "2 훈련 MSE: 0.004675172\n",
      "3 훈련 MSE: 0.004376106\n"
     ]
    }
   ],
   "source": [
    "hidden_output, W1, b1, W4, b4 = train_autoencoder(mnist.train.images, n_neurons=300, n_epochs=4, batch_size=150,\n",
    "                                                  output_activation=None)\n",
    "_, W2, b2, W3, b3 = train_autoencoder(hidden_output, n_neurons=150, n_epochs=4, batch_size=150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "마지막으로 방금전 훈련한 오토인코더의 가중치와 편향을 재사용하여 적층 오토인코더를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden1 = tf.nn.elu(tf.matmul(X, W1) + b1)\n",
    "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n",
    "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n",
    "outputs = tf.matmul(hidden3, W4) + b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9400 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 하나의 그래프에서 오토인코더를 따로따로 훈련하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하나의 그래프를 사용하는 방법도 있습니다. 이 방법은 전체 적층 오토인코더를 위한 그래프를 만들지만 각 오토인코더를 독립적으로 훈련하기 위한 연산도 추가합니다. 단계 1은 맨 아래층과 맨 윗층을 훈련하고(즉, 첫 번째 오토인코더), 단계 2는 두 개의 가운데 층을 훈련합니다(즉, 두 번째 오토인코더)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "weights4_init = initializer([n_hidden3, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "weights4 = tf.Variable(weights4_init, dtype=tf.float32, name=\"weights4\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "with tf.name_scope(\"phase1\"):\n",
    "    phase1_outputs = tf.matmul(hidden1, weights4) + biases4  # hidden2와 hidden3 통과합니다\n",
    "    phase1_reconstruction_loss = tf.reduce_mean(tf.square(phase1_outputs - X))\n",
    "    phase1_reg_loss = regularizer(weights1) + regularizer(weights4)\n",
    "    phase1_loss = phase1_reconstruction_loss + phase1_reg_loss\n",
    "    phase1_training_op = optimizer.minimize(phase1_loss)\n",
    "\n",
    "with tf.name_scope(\"phase2\"):\n",
    "    phase2_reconstruction_loss = tf.reduce_mean(tf.square(hidden3 - hidden1))\n",
    "    phase2_reg_loss = regularizer(weights2) + regularizer(weights3)\n",
    "    phase2_loss = phase2_reconstruction_loss + phase2_reg_loss\n",
    "    train_vars = [weights2, biases2, weights3, biases3]\n",
    "    phase2_training_op = optimizer.minimize(phase2_loss, var_list=train_vars) # hidden1 동결"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "훈련 단계 #1\n",
      "0 훈련 MSE: 0.007406866\n",
      "1 훈련 MSE: 0.0078288205\n",
      "2 훈련 MSE: 0.007728097\n",
      "3 훈련 MSE: 0.0074090064\n",
      "훈련 단계 #2\n",
      "0 훈련 MSE: 0.35758996\n",
      "1 훈련 MSE: 0.00593613\n",
      "2 훈련 MSE: 0.0030089526\n",
      "3 훈련 MSE: 0.0024365915\n",
      "테스트 MSE: 0.009815396\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op]\n",
    "reconstruction_losses = [phase1_reconstruction_loss, phase2_reconstruction_loss]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"훈련 단계 #{}\".format(phase + 1))\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                sess.run(training_ops[phase], feed_dict={X: X_batch})\n",
    "            loss_train = reconstruction_losses[phase].eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "            saver.save(sess, \"./my_model_one_at_a_time.ckpt\")\n",
    "    loss_test = reconstruction_loss.eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"테스트 MSE:\", loss_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 동결 층의 출력을 캐싱하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "훈련 단계 #1\n",
      "0 훈련 MSE: 0.0075382586\n",
      "1 훈련 MSE: 0.007754701\n",
      "2 훈련 MSE: 0.007343679\n",
      "3 훈련 MSE: 0.007837775\n",
      "훈련 단계 #2\n",
      "0 훈련 MSE: 0.18597357\n",
      "1 훈련 MSE: 0.0044647465\n",
      "2 훈련 MSE: 0.002462252\n",
      "3 훈련 MSE: 0.0020415403\n",
      "테스트 MSE: 0.009790834\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op]\n",
    "reconstruction_losses = [phase1_reconstruction_loss, phase2_reconstruction_loss]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"훈련 단계 #{}\".format(phase + 1))\n",
    "        if phase == 1:\n",
    "            hidden1_cache = hidden1.eval(feed_dict={X: mnist.train.images})\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                if phase == 1:\n",
    "                    indices = rnd.permutation(mnist.train.num_examples)\n",
    "                    hidden1_batch = hidden1_cache[indices[:batch_sizes[phase]]]\n",
    "                    feed_dict = {hidden1: hidden1_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "                else:\n",
    "                    X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                    feed_dict = {X: X_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "            loss_train = reconstruction_losses[phase].eval(feed_dict=feed_dict)\n",
    "            print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "            saver.save(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    loss_test = reconstruction_loss.eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"테스트 MSE:\", loss_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 재구성 시각화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_one_at_a_time.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 7904 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_test_digits = 2\n",
    "X_test = mnist.test.images[:n_test_digits]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_one_at_a_time.ckpt\") # not shown in the book\n",
    "    outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "for digit_index in range(n_test_digits):\n",
    "    plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "    plot_image(X_test[digit_index])\n",
    "    plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "    plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 특성 시각화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_one_at_a_time.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 10836 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_one_at_a_time.ckpt\") # 책에는 없음\n",
    "    weights1_val = weights1.eval()\n",
    "\n",
    "for i in range(5):\n",
    "    plt.subplot(1, 5, i + 1)\n",
    "    plot_image(weights1_val.T[i])\n",
    "\n",
    "save_fig(\"extracted_features_plot\") # 책에는 없음\n",
    "plt.show()                          # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 비지도 사전훈련"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST 분류 문제를 위한 작은 신경망을 만들겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_outputs), name=\"biases3\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "logits = tf.matmul(hidden2, weights3) + biases3\n",
    "\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2) + regularizer(weights3)\n",
    "loss = cross_entropy + reg_loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "pretrain_saver = tf.train.Saver([weights1, weights2, biases1, biases2])\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(사전훈련 없이)평범하게 훈련시킵니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 정확도: 0.97333336 테스트 정확도: 0.9334\n",
      "1 훈련 정확도: 0.98 테스트 정확도: 0.936\n",
      "2 훈련 정확도: 0.97333336 테스트 정확도: 0.9382\n",
      "3 훈련 정확도: 0.9866667 테스트 정확도: 0.9489\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 정확도:\", accuracy_val, end=\" \")\n",
    "        saver.save(sess, \"./my_model_supervised.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"테스트 정확도:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "사전 훈련된 오토인코더의 첫 두개의 층을 재사용해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_cache_frozen.ckpt\n",
      "0 훈련 정확도: 0.9533333\t테스트 정확도: 0.9294\n",
      "1 훈련 정확도: 0.96666664\t테스트 정확도: 0.9342\n",
      "2 훈련 정확도: 0.98\t테스트 정확도: 0.9449\n",
      "3 훈련 정확도: 0.9866667\t테스트 정확도: 0.9462\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "#training_op = optimizer.minimize(loss, var_list=[weights3, biases3])  # layers 1와 2를 동결 (선택사항)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    pretrain_saver.restore(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 정확도:\", accuracy_val, end=\"\\t\")\n",
    "        saver.save(sess, \"./my_model_supervised_pretrained.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"테스트 정확도:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 적층 잡음제거 오토인코더"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "가우시안 잡음을 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise_level = 1.0\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "X_noisy = X + noise_level * tf.random_normal(tf.shape(X))\n",
    "\n",
    "hidden1 = tf.layers.dense(X_noisy, n_hidden1, activation=tf.nn.relu,\n",
    "                          name=\"hidden1\")\n",
    "hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden2\")                            # 책에는 없음\n",
    "hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden3\")                            # 책에는 없음\n",
    "outputs = tf.layers.dense(hidden3, n_outputs, name=\"outputs\")        # 책에는 없음\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X)) # MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(reconstruction_loss)\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.046190277\n",
      "1 훈련 MSE: 0.044343986\n",
      "2 훈련 MSE: 0.04200526\n",
      "3 훈련 MSE: 0.040973965\n",
      "4 훈련 MSE: 0.040393338\n",
      "5 훈련 MSE: 0.038924493\n",
      "6 훈련 MSE: 0.0394448\n",
      "7 훈련 MSE: 0.042411502\n",
      "8 훈련 MSE: 0.039848443\n",
      "9 훈련 MSE: 0.04152445\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising_gaussian.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "드롭아웃을 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropout_rate = 0.3\n",
    "\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "X_drop = tf.layers.dropout(X, dropout_rate, training=training)\n",
    "\n",
    "hidden1 = tf.layers.dense(X_drop, n_hidden1, activation=tf.nn.relu,\n",
    "                          name=\"hidden1\")\n",
    "hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden2\")                            # 책에는 없음\n",
    "hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden3\")                            # 책에는 없음\n",
    "outputs = tf.layers.dense(hidden3, n_outputs, name=\"outputs\")        # 책에는 없음\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X)) # MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(reconstruction_loss)\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.027500158\n",
      "1 훈련 MSE: 0.02639269\n",
      "2 훈련 MSE: 0.022876455\n",
      "3 훈련 MSE: 0.0229337\n",
      "4 훈련 MSE: 0.022025634\n",
      "5 훈련 MSE: 0.021996608\n",
      "6 훈련 MSE: 0.021385597\n",
      "7 훈련 MSE: 0.021463407\n",
      "8 훈련 MSE: 0.021037336\n",
      "9 훈련 MSE: 0.02082274\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, training: True})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising_dropout.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_stacked_denoising_dropout.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 9484 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_stacked_denoising_dropout.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 희소 오토인코더"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 21616 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = 0.1\n",
    "q = np.linspace(0.001, 0.999, 500)\n",
    "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
    "mse = (p - q)**2\n",
    "plt.plot([p, p], [0, 0.3], \"k:\")\n",
    "plt.text(0.05, 0.32, \"목표 희소\", fontsize=14)\n",
    "plt.plot(q, kl_div, \"b-\", label=\"쿨백 라이블러 발산\")\n",
    "plt.plot(q, mse, \"r--\", label=\"MSE\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"실제 희소\")\n",
    "plt.ylabel(\"비용\", rotation=0)\n",
    "plt.axis([0, 1, 0, 0.95])\n",
    "save_fig(\"sparsity_loss_plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 1000  # 희소 코딩 유닛\n",
    "n_outputs = n_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kl_divergence(p, q):\n",
    "    # 쿨백 라이블러 발산\n",
    "    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))\n",
    "\n",
    "learning_rate = 0.01\n",
    "sparsity_target = 0.1\n",
    "sparsity_weight = 0.2\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])            # 책에는 없음\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.sigmoid) # 책에는 없음\n",
    "outputs = tf.layers.dense(hidden1, n_outputs)                     # 책에는 없음\n",
    "\n",
    "hidden1_mean = tf.reduce_mean(hidden1, axis=0) # 배치 평균\n",
    "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X)) # MSE\n",
    "loss = reconstruction_loss + sparsity_weight * sparsity_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.13819474 \t희소 손실: 0.41844434 \t전체 손실: 0.22188361\n",
      "1 훈련 MSE: 0.059185907 \t희소 손실: 0.010747161 \t전체 손실: 0.06133534\n",
      "2 훈련 MSE: 0.053931113 \t희소 손실: 0.019957827 \t전체 손실: 0.05792268\n",
      "3 훈련 MSE: 0.04771372 \t희소 손실: 0.039470375 \t전체 손실: 0.055607796\n",
      "4 훈련 MSE: 0.044841796 \t희소 손실: 0.011593379 \t전체 손실: 0.047160473\n",
      "5 훈련 MSE: 0.04044193 \t희소 손실: 0.09246979 \t전체 손실: 0.058935888\n",
      "6 훈련 MSE: 0.03886659 \t희소 손실: 0.045811094 \t전체 손실: 0.04802881\n",
      "7 훈련 MSE: 0.037868652 \t희소 손실: 0.07498116 \t전체 손실: 0.052864887\n",
      "8 훈련 MSE: 0.033122517 \t희소 손실: 0.020321798 \t전체 손실: 0.037186876\n",
      "9 훈련 MSE: 0.031449173 \t희소 손실: 0.095670365 \t전체 손실: 0.050583243\n",
      "10 훈련 MSE: 0.027398113 \t희소 손실: 0.065706335 \t전체 손실: 0.04053938\n",
      "11 훈련 MSE: 0.024734963 \t희소 손실: 0.08889374 \t전체 손실: 0.042513713\n",
      "12 훈련 MSE: 0.023367295 \t희소 손실: 0.0576614 \t전체 손실: 0.034899574\n",
      "13 훈련 MSE: 0.023007901 \t희소 손실: 0.061935436 \t전체 손실: 0.03539499\n",
      "14 훈련 MSE: 0.02123353 \t희소 손실: 0.025881436 \t전체 손실: 0.026409818\n",
      "15 훈련 MSE: 0.02206469 \t희소 손실: 0.48820436 \t전체 손실: 0.11970556\n",
      "16 훈련 MSE: 0.01910253 \t희소 손실: 0.032202266 \t전체 손실: 0.025542984\n",
      "17 훈련 MSE: 0.01898219 \t희소 손실: 0.12995036 \t전체 손실: 0.044972263\n",
      "18 훈련 MSE: 0.017487913 \t희소 손실: 0.035002217 \t전체 손실: 0.024488356\n",
      "19 훈련 MSE: 0.017776337 \t희소 손실: 0.11024549 \t전체 손실: 0.039825432\n",
      "20 훈련 MSE: 0.016910141 \t희소 손실: 0.030320741 \t전체 손실: 0.02297429\n",
      "21 훈련 MSE: 0.018679725 \t희소 손실: 0.36376768 \t전체 손실: 0.091433264\n",
      "22 훈련 MSE: 0.016285753 \t희소 손실: 0.055482324 \t전체 손실: 0.027382217\n",
      "23 훈련 MSE: 0.015950203 \t희소 손실: 0.079365596 \t전체 손실: 0.031823322\n",
      "24 훈련 MSE: 0.015672017 \t희소 손실: 0.14289412 \t전체 손실: 0.04425084\n",
      "25 훈련 MSE: 0.014612305 \t희소 손실: 0.13330609 \t전체 손실: 0.041273523\n",
      "26 훈련 MSE: 0.014524785 \t희소 손실: 0.10948229 \t전체 손실: 0.036421243\n",
      "27 훈련 MSE: 0.013598451 \t희소 손실: 0.07208246 \t전체 손실: 0.028014943\n",
      "28 훈련 MSE: 0.014069845 \t희소 손실: 0.15789555 \t전체 손실: 0.045648955\n",
      "29 훈련 MSE: 0.013441612 \t희소 손실: 0.14449221 \t전체 손실: 0.042340055\n",
      "30 훈련 MSE: 0.013608929 \t희소 손실: 0.18507166 \t전체 손실: 0.05062326\n",
      "31 훈련 MSE: 0.014197283 \t희소 손실: 0.06788807 \t전체 손실: 0.027774898\n",
      "32 훈련 MSE: 0.013759435 \t희소 손실: 0.057576656 \t전체 손실: 0.025274767\n",
      "33 훈련 MSE: 0.013667367 \t희소 손실: 0.11363616 \t전체 손실: 0.0363946\n",
      "34 훈련 MSE: 0.0126469685 \t희소 손실: 0.13469556 \t전체 손실: 0.039586082\n",
      "35 훈련 MSE: 0.012872889 \t희소 손실: 0.16850437 \t전체 손실: 0.046573766\n",
      "36 훈련 MSE: 0.012669464 \t희소 손실: 0.09313129 \t전체 손실: 0.03129572\n",
      "37 훈련 MSE: 0.012427666 \t희소 손실: 0.105387196 \t전체 손실: 0.033505104\n",
      "38 훈련 MSE: 0.01233604 \t희소 손실: 0.19235855 \t전체 손실: 0.05080775\n",
      "39 훈련 MSE: 0.012337869 \t희소 손실: 0.14231703 \t전체 손실: 0.040801276\n",
      "40 훈련 MSE: 0.012626693 \t희소 손실: 0.23267308 \t전체 손실: 0.05916131\n",
      "41 훈련 MSE: 0.012383292 \t희소 손실: 0.104892224 \t전체 손실: 0.033361737\n",
      "42 훈련 MSE: 0.011781142 \t희소 손실: 0.10161172 \t전체 손실: 0.032103486\n",
      "43 훈련 MSE: 0.012044032 \t희소 손실: 0.1359405 \t전체 손실: 0.039232135\n",
      "44 훈련 MSE: 0.011620717 \t희소 손실: 0.11472766 \t전체 손실: 0.03456625\n",
      "45 훈련 MSE: 0.011669101 \t희소 손실: 0.15798101 \t전체 손실: 0.043265305\n",
      "46 훈련 MSE: 0.01167006 \t희소 손실: 0.13549636 \t전체 손실: 0.038769335\n",
      "47 훈련 MSE: 0.011549011 \t희소 손실: 0.15662336 \t전체 손실: 0.042873684\n",
      "48 훈련 MSE: 0.011729614 \t희소 손실: 0.4430182 \t전체 손실: 0.10033325\n",
      "49 훈련 MSE: 0.011562036 \t희소 손실: 0.17581894 \t전체 손실: 0.046725824\n",
      "50 훈련 MSE: 0.01135992 \t희소 손실: 0.18881696 \t전체 손실: 0.049123313\n",
      "51 훈련 MSE: 0.0115385335 \t희소 손실: 0.26335293 \t전체 손실: 0.06420912\n",
      "52 훈련 MSE: 0.0120425625 \t희소 손실: 0.3675837 \t전체 손실: 0.0855593\n",
      "53 훈련 MSE: 0.011238733 \t희소 손실: 0.12488687 \t전체 손실: 0.03621611\n",
      "54 훈련 MSE: 0.013967819 \t희소 손실: 0.43314826 \t전체 손실: 0.10059747\n",
      "55 훈련 MSE: 0.012766779 \t희소 손실: 0.07719517 \t전체 손실: 0.028205812\n",
      "56 훈련 MSE: 0.012239235 \t희소 손실: 0.30152494 \t전체 손실: 0.072544225\n",
      "57 훈련 MSE: 0.012724373 \t희소 손실: 0.24817757 \t전체 손실: 0.062359888\n",
      "58 훈련 MSE: 0.023815982 \t희소 손실: 0.2323372 \t전체 손실: 0.07028343\n",
      "59 훈련 MSE: 0.013152219 \t희소 손실: 0.19813177 \t전체 손실: 0.052778576\n",
      "60 훈련 MSE: 0.019204749 \t희소 손실: 0.19425029 \t전체 손실: 0.058054805\n",
      "61 훈련 MSE: 0.01287946 \t희소 손실: 0.22912267 \t전체 손실: 0.058703996\n",
      "62 훈련 MSE: 0.014201862 \t희소 손실: 0.47936517 \t전체 손실: 0.1100749\n",
      "63 훈련 MSE: 0.014247858 \t희소 손실: 0.3340563 \t전체 손실: 0.08105911\n",
      "64 훈련 MSE: 0.01188848 \t희소 손실: 0.17818896 \t전체 손실: 0.04752627\n",
      "65 훈련 MSE: 0.016912188 \t희소 손실: 0.18625271 \t전체 손실: 0.054162733\n",
      "66 훈련 MSE: 0.015350101 \t희소 손실: 0.984547 \t전체 손실: 0.21225952\n",
      "67 훈련 MSE: 0.037533756 \t희소 손실: 0.31189525 \t전체 손실: 0.09991281\n",
      "68 훈련 MSE: 0.015359703 \t희소 손실: 0.21824294 \t전체 손실: 0.059008293\n",
      "69 훈련 MSE: 0.0130188465 \t희소 손실: 0.43986538 \t전체 손실: 0.10099193\n",
      "70 훈련 MSE: 0.04249404 \t희소 손실: 0.7454435 \t전체 손실: 0.19158275\n",
      "71 훈련 MSE: 0.014368004 \t희소 손실: 0.24009731 \t전체 손실: 0.06238747\n",
      "72 훈련 MSE: 0.015802076 \t희소 손실: 0.7972901 \t전체 손실: 0.1752601\n",
      "73 훈련 MSE: 0.014027514 \t희소 손실: 0.31637588 \t전체 손실: 0.077302694\n",
      "74 훈련 MSE: 0.033579413 \t희소 손실: 0.32424852 \t전체 손실: 0.09842911\n",
      "75 훈련 MSE: 0.037583392 \t희소 손실: 0.51026374 \t전체 손실: 0.13963614\n",
      "76 훈련 MSE: 0.018015897 \t희소 손실: 0.19957174 \t전체 손실: 0.057930246\n",
      "77 훈련 MSE: 0.019651866 \t희소 손실: 0.30766442 \t전체 손실: 0.08118475\n",
      "78 훈련 MSE: 0.018113064 \t희소 손실: 0.5368116 \t전체 손실: 0.12547539\n",
      "79 훈련 MSE: 0.019586327 \t희소 손실: 0.4225994 \t전체 손실: 0.10410621\n",
      "80 훈련 MSE: 0.016286958 \t희소 손실: 0.116765514 \t전체 손실: 0.03964006\n",
      "81 훈련 MSE: 0.012678387 \t희소 손실: 0.8773603 \t전체 손실: 0.18815045\n",
      "82 훈련 MSE: 0.015514646 \t희소 손실: 2.136949 \t전체 손실: 0.44290447\n",
      "83 훈련 MSE: 0.014462668 \t희소 손실: 0.6624632 \t전체 손실: 0.14695531\n",
      "84 훈련 MSE: 0.0139609 \t희소 손실: 0.2704364 \t전체 손실: 0.06804818\n",
      "85 훈련 MSE: 0.02716384 \t희소 손실: 0.93438864 \t전체 손실: 0.21404156\n",
      "86 훈련 MSE: 0.014590265 \t희소 손실: 0.6826812 \t전체 손실: 0.1511265\n",
      "87 훈련 MSE: 0.012223512 \t희소 손실: 1.41284 \t전체 손실: 0.29479152\n",
      "88 훈련 MSE: 0.01925102 \t희소 손실: 0.8619595 \t전체 손실: 0.19164293\n",
      "89 훈련 MSE: 0.029023234 \t희소 손실: 0.2942158 \t전체 손실: 0.087866396\n",
      "90 훈련 MSE: 0.03109355 \t희소 손실: 0.45987695 \t전체 손실: 0.12306894\n",
      "91 훈련 MSE: 0.014538759 \t희소 손실: 0.09905543 \t전체 손실: 0.034349844\n",
      "92 훈련 MSE: 0.015151837 \t희소 손실: 0.38795644 \t전체 손실: 0.09274313\n",
      "93 훈련 MSE: 0.01682431 \t희소 손실: 0.1613949 \t전체 손실: 0.04910329\n",
      "94 훈련 MSE: 0.011993256 \t희소 손실: 0.11752173 \t전체 손실: 0.035497602\n",
      "95 훈련 MSE: 0.0139113525 \t희소 손실: 0.10376862 \t전체 손실: 0.034665078\n",
      "96 훈련 MSE: 0.018343525 \t희소 손실: 0.23694368 \t전체 손실: 0.06573226\n",
      "97 훈련 MSE: 0.013478864 \t희소 손실: 0.10932048 \t전체 손실: 0.03534296\n",
      "98 훈련 MSE: 0.013734443 \t희소 손실: 0.089936554 \t전체 손실: 0.031721756\n",
      "99 훈련 MSE: 0.0233883 \t희소 손실: 0.33332965 \t전체 손실: 0.09005423\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 1000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        reconstruction_loss_val, sparsity_loss_val, loss_val = sess.run([reconstruction_loss, sparsity_loss, loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", reconstruction_loss_val, \"\\t희소 손실:\", sparsity_loss_val, \"\\t전체 손실:\", loss_val)\n",
    "        saver.save(sess, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_sparse.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 10228 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "코딩층은 0에서 1사이의 값을 출력해야 하므로 시그모이드 활성화 함수를 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.sigmoid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훈련 속도를 높이기 위해 입력을 0과 1사이로 정규화하고 비용 함수로 MSE 대신 크로스엔트로피를 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = tf.layers.dense(hidden1, n_outputs)\n",
    "outputs = tf.nn.sigmoid(logits)\n",
    "\n",
    "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits)\n",
    "reconstruction_loss = tf.reduce_mean(xentropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 변이형 오토인코더"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # 코딩 유닛\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "learning_rate = 0.001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "my_dense_layer = partial(\n",
    "    tf.layers.dense,\n",
    "    activation=tf.nn.elu,\n",
    "    kernel_initializer=initializer)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3_mean = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "hidden3_sigma = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "noise = tf.random_normal(tf.shape(hidden3_sigma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + hidden3_sigma * noise\n",
    "hidden4 = my_dense_layer(hidden3, n_hidden4)\n",
    "hidden5 = my_dense_layer(hidden4, n_hidden5)\n",
    "logits = my_dense_layer(hidden5, n_outputs, activation=None)\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits)\n",
    "reconstruction_loss = tf.reduce_sum(xentropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps = 1e-10 # NaN을 반환하는 log(0)을 피하기 위한 안전항\n",
    "latent_loss = 0.5 * tf.reduce_sum(\n",
    "    tf.square(hidden3_sigma) + tf.square(hidden3_mean)\n",
    "    - 1 - tf.log(eps + tf.square(hidden3_sigma)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 전체 손실: 33711.305 \t재구성 손실: 24141.234 \t잠재 손실: 9570.07\n",
      "1 훈련 전체 손실: 27055.078 \t재구성 손실: 22278.68 \t잠재 손실: 4776.3984\n",
      "2 훈련 전체 손실: 28769.135 \t재구성 손실: 23155.71 \t잠재 손실: 5613.4233\n",
      "3 훈련 전체 손실: 26304.252 \t재구성 손실: 21337.469 \t잠재 손실: 4966.783\n",
      "4 훈련 전체 손실: 31762.32 \t재구성 손실: 24771.611 \t잠재 손실: 6990.71\n",
      "5 훈련 전체 손실: 25975.902 \t재구성 손실: 21666.244 \t잠재 손실: 4309.659\n",
      "6 훈련 전체 손실: 21084.105 \t재구성 손실: 17912.25 \t잠재 손실: 3171.8557\n",
      "7 훈련 전체 손실: 20691.05 \t재구성 손실: 17767.701 \t잠재 손실: 2923.3506\n",
      "8 훈련 전체 손실: 18301.613 \t재구성 손실: 15335.816 \t잠재 손실: 2965.7979\n",
      "9 훈련 전체 손실: 18003.305 \t재구성 손실: 14857.625 \t잠재 손실: 3145.6807\n",
      "10 훈련 전체 손실: 16868.87 \t재구성 손실: 13742.127 \t잠재 손실: 3126.7417\n",
      "11 훈련 전체 손실: 16322.584 \t재구성 손실: 13176.514 \t잠재 손실: 3146.0708\n",
      "12 훈련 전체 손실: 16848.775 \t재구성 손실: 13641.574 \t잠재 손실: 3207.201\n",
      "13 훈련 전체 손실: 16411.53 \t재구성 손실: 13193.485 \t잠재 손실: 3218.0444\n",
      "14 훈련 전체 손실: 16841.875 \t재구성 손실: 13629.753 \t잠재 손실: 3212.122\n",
      "15 훈련 전체 손실: 18734.246 \t재구성 손실: 15469.811 \t잠재 손실: 3264.4355\n",
      "16 훈련 전체 손실: 15549.877 \t재구성 손실: 12475.354 \t잠재 손실: 3074.523\n",
      "17 훈련 전체 손실: 16549.727 \t재구성 손실: 13114.041 \t잠재 손실: 3435.6865\n",
      "18 훈련 전체 손실: 16204.832 \t재구성 손실: 12911.634 \t잠재 손실: 3293.198\n",
      "19 훈련 전체 손실: 19072.146 \t재구성 손실: 15676.701 \t잠재 손실: 3395.4456\n",
      "20 훈련 전체 손실: 18689.686 \t재구성 손실: 14211.309 \t잠재 손실: 4478.3774\n",
      "21 훈련 전체 손실: 16398.697 \t재구성 손실: 13045.562 \t잠재 손실: 3353.135\n",
      "22 훈련 전체 손실: 16106.504 \t재구성 손실: 12735.91 \t잠재 손실: 3370.5942\n",
      "23 훈련 전체 손실: 20430.426 \t재구성 손실: 15874.012 \t잠재 손실: 4556.413\n",
      "24 훈련 전체 손실: 15652.191 \t재구성 손실: 12415.114 \t잠재 손실: 3237.077\n",
      "25 훈련 전체 손실: 15460.601 \t재구성 손실: 12243.658 \t잠재 손실: 3216.9421\n",
      "26 훈련 전체 손실: 15832.671 \t재구성 손실: 12465.408 \t잠재 손실: 3367.2627\n",
      "27 훈련 전체 손실: 37082.61 \t재구성 손실: 25035.375 \t잠재 손실: 12047.233\n",
      "28 훈련 전체 손실: 26629.25 \t재구성 손실: 21904.469 \t잠재 손실: 4724.7803\n",
      "29 훈련 전체 손실: 23798.465 \t재구성 손실: 19210.568 \t잠재 손실: 4587.8955\n",
      "30 훈련 전체 손실: 20755.977 \t재구성 손실: 17576.453 \t잠재 손실: 3179.5234\n",
      "31 훈련 전체 손실: 17813.498 \t재구성 손실: 14805.74 \t잠재 손실: 3007.7578\n",
      "32 훈련 전체 손실: 17064.127 \t재구성 손실: 13854.3125 \t잠재 손실: 3209.8142\n",
      "33 훈련 전체 손실: 16516.027 \t재구성 손실: 13270.247 \t잠재 손실: 3245.781\n",
      "34 훈련 전체 손실: 16060.129 \t재구성 손실: 12752.525 \t잠재 손실: 3307.6035\n",
      "35 훈련 전체 손실: 16070.643 \t재구성 손실: 12676.658 \t잠재 손실: 3393.9841\n",
      "36 훈련 전체 손실: 16754.852 \t재구성 손실: 13328.36 \t잠재 손실: 3426.492\n",
      "37 훈련 전체 손실: 16162.049 \t재구성 손실: 12813.662 \t잠재 손실: 3348.3872\n",
      "38 훈련 전체 손실: 16215.911 \t재구성 손실: 12828.523 \t잠재 손실: 3387.388\n",
      "39 훈련 전체 손실: 15912.985 \t재구성 손실: 12628.198 \t잠재 손실: 3284.787\n",
      "40 훈련 전체 손실: 15275.752 \t재구성 손실: 11982.837 \t잠재 손실: 3292.9146\n",
      "41 훈련 전체 손실: 16066.625 \t재구성 손실: 12757.855 \t잠재 손실: 3308.7693\n",
      "42 훈련 전체 손실: 17823.701 \t재구성 손실: 14552.526 \t잠재 손실: 3271.1743\n",
      "43 훈련 전체 손실: 24408.918 \t재구성 손실: 19846.832 \t잠재 손실: 4562.0864\n",
      "44 훈련 전체 손실: 24694.812 \t재구성 손실: 19852.523 \t잠재 손실: 4842.288\n",
      "45 훈련 전체 손실: 30566.844 \t재구성 손실: 22793.793 \t잠재 손실: 7773.05\n",
      "46 훈련 전체 손실: 24413.105 \t재구성 손실: 19724.02 \t잠재 손실: 4689.086\n",
      "47 훈련 전체 손실: 23234.568 \t재구성 손실: 18196.725 \t잠재 손실: 5037.8438\n",
      "48 훈련 전체 손실: 17091.879 \t재구성 손실: 13951.564 \t잠재 손실: 3140.3154\n",
      "49 훈련 전체 손실: 15979.056 \t재구성 손실: 12713.727 \t잠재 손실: 3265.3289\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_val, reconstruction_loss_val, latent_loss_val = sess.run([loss, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 전체 손실:\", loss_val, \"\\t재구성 손실:\", reconstruction_loss_val, \"\\t잠재 손실:\", latent_loss_val)\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # 코딩 유닛\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "learning_rate = 0.001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "my_dense_layer = partial(\n",
    "    tf.layers.dense,\n",
    "    activation=tf.nn.elu,\n",
    "    kernel_initializer=initializer)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3_mean = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "hidden3_gamma = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "noise = tf.random_normal(tf.shape(hidden3_gamma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + tf.exp(0.5 * hidden3_gamma) * noise\n",
    "hidden4 = my_dense_layer(hidden3, n_hidden4)\n",
    "hidden5 = my_dense_layer(hidden4, n_hidden5)\n",
    "logits = my_dense_layer(hidden5, n_outputs, activation=None)\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits)\n",
    "reconstruction_loss = tf.reduce_sum(xentropy)\n",
    "latent_loss = 0.5 * tf.reduce_sum(\n",
    "    tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)\n",
    "loss = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 숫자 이미지 생성"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "모델을 훈련시켜 랜덤한 이미지를 생성해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 전체 손실: 17845.31 \t재구성 손실: 14199.457 \t잠재 손실: 3645.853\n",
      "1 훈련 전체 손실: 17223.674 \t재구성 손실: 13457.104 \t잠재 손실: 3766.5708\n",
      "2 훈련 전체 손실: 16330.138 \t재구성 손실: 12578.04 \t잠재 손실: 3752.098\n",
      "3 훈련 전체 손실: 16323.255 \t재구성 손실: 12578.617 \t잠재 손실: 3744.638\n",
      "4 훈련 전체 손실: 16297.912 \t재구성 손실: 12533.176 \t잠재 손실: 3764.7368\n",
      "5 훈련 전체 손실: 15681.15 \t재구성 손실: 11960.288 \t잠재 손실: 3720.862\n",
      "6 훈련 전체 손실: 16120.67 \t재구성 손실: 12325.412 \t잠재 손실: 3795.258\n",
      "7 훈련 전체 손실: 15783.96 \t재구성 손실: 12021.791 \t잠재 손실: 3762.169\n",
      "8 훈련 전체 손실: 16208.828 \t재구성 손실: 12365.744 \t잠재 손실: 3843.0837\n",
      "9 훈련 전체 손실: 15524.169 \t재구성 손실: 11801.077 \t잠재 손실: 3723.092\n",
      "10 훈련 전체 손실: 16092.582 \t재구성 손실: 12279.52 \t잠재 손실: 3813.0625\n",
      "11 훈련 전체 손실: 16111.75 \t재구성 손실: 12282.848 \t잠재 손실: 3828.9019\n",
      "12 훈련 전체 손실: 15926.159 \t재구성 손실: 12180.764 \t잠재 손실: 3745.3953\n",
      "13 훈련 전체 손실: 15271.382 \t재구성 손실: 11631.805 \t잠재 손실: 3639.5774\n",
      "14 훈련 전체 손실: 16211.1875 \t재구성 손실: 12341.906 \t잠재 손실: 3869.2817\n",
      "15 훈련 전체 손실: 15781.817 \t재구성 손실: 11920.02 \t잠재 손실: 3861.798\n",
      "16 훈련 전체 손실: 15513.055 \t재구성 손실: 11780.989 \t잠재 손실: 3732.0657\n",
      "17 훈련 전체 손실: 15282.611 \t재구성 손실: 11568.492 \t잠재 손실: 3714.1196\n",
      "18 훈련 전체 손실: 15077.107 \t재구성 손실: 11422.83 \t잠재 손실: 3654.277\n",
      "19 훈련 전체 손실: 15184.711 \t재구성 손실: 11399.879 \t잠재 손실: 3784.8323\n",
      "20 훈련 전체 손실: 15317.0625 \t재구성 손실: 11541.074 \t잠재 손실: 3775.988\n",
      "21 훈련 전체 손실: 15375.975 \t재구성 손실: 11662.023 \t잠재 손실: 3713.9507\n",
      "22 훈련 전체 손실: 15253.931 \t재구성 손실: 11569.6 \t잠재 손실: 3684.3308\n",
      "23 훈련 전체 손실: 14920.597 \t재구성 손실: 11226.293 \t잠재 손실: 3694.304\n",
      "24 훈련 전체 손실: 15152.121 \t재구성 손실: 11461.097 \t잠재 손실: 3691.0242\n",
      "25 훈련 전체 손실: 15161.9795 \t재구성 손실: 11527.926 \t잠재 손실: 3634.054\n",
      "26 훈련 전체 손실: 15351.327 \t재구성 손실: 11591.437 \t잠재 손실: 3759.8906\n",
      "27 훈련 전체 손실: 15188.266 \t재구성 손실: 11508.098 \t잠재 손실: 3680.168\n",
      "28 훈련 전체 손실: 14944.834 \t재구성 손실: 11275.673 \t잠재 손실: 3669.1616\n",
      "29 훈련 전체 손실: 14949.296 \t재구성 손실: 11173.085 \t잠재 손실: 3776.211\n",
      "30 훈련 전체 손실: 14824.383 \t재구성 손실: 11233.498 \t잠재 손실: 3590.8848\n",
      "31 훈련 전체 손실: 15053.771 \t재구성 손실: 11336.643 \t잠재 손실: 3717.1284\n",
      "32 훈련 전체 손실: 15209.582 \t재구성 손실: 11480.851 \t잠재 손실: 3728.731\n",
      "33 훈련 전체 손실: 15029.901 \t재구성 손실: 11239.151 \t잠재 손실: 3790.75\n",
      "34 훈련 전체 손실: 15018.421 \t재구성 손실: 11247.266 \t잠재 손실: 3771.1555\n",
      "35 훈련 전체 손실: 15295.654 \t재구성 손실: 11592.32 \t잠재 손실: 3703.334\n",
      "36 훈련 전체 손실: 15067.271 \t재구성 손실: 11287.572 \t잠재 손실: 3779.6997\n",
      "37 훈련 전체 손실: 14845.694 \t재구성 손실: 11252.967 \t잠재 손실: 3592.7273\n",
      "38 훈련 전체 손실: 14872.638 \t재구성 손실: 11183.228 \t잠재 손실: 3689.4102\n",
      "39 훈련 전체 손실: 14961.719 \t재구성 손실: 11297.793 \t잠재 손실: 3663.9263\n",
      "40 훈련 전체 손실: 14667.465 \t재구성 손실: 10977.892 \t잠재 손실: 3689.5732\n",
      "41 훈련 전체 손실: 15164.336 \t재구성 손실: 11456.284 \t잠재 손실: 3708.0515\n",
      "42 훈련 전체 손실: 14535.691 \t재구성 손실: 10862.055 \t잠재 손실: 3673.6365\n",
      "43 훈련 전체 손실: 14729.365 \t재구성 손실: 11080.487 \t잠재 손실: 3648.8777\n",
      "44 훈련 전체 손실: 15044.16 \t재구성 손실: 11426.916 \t잠재 손실: 3617.2446\n",
      "45 훈련 전체 손실: 14839.276 \t재구성 손실: 11058.229 \t잠재 손실: 3781.0476\n",
      "46 훈련 전체 손실: 14890.201 \t재구성 손실: 11144.563 \t잠재 손실: 3745.6377\n",
      "47 훈련 전체 손실: 14605.932 \t재구성 손실: 10951.763 \t잠재 손실: 3654.1694\n",
      "48 훈련 전체 손실: 14460.681 \t재구성 손실: 10878.129 \t잠재 손실: 3582.552\n",
      "49 훈련 전체 손실: 14584.321 \t재구성 손실: 10932.154 \t잠재 손실: 3652.1667\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "n_digits = 60\n",
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\") # not shown in the book\n",
    "            sys.stdout.flush()                                          # not shown\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_val, reconstruction_loss_val, latent_loss_val = sess.run([loss, reconstruction_loss, latent_loss], feed_dict={X: X_batch}) # not shown\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 전체 손실:\", loss_val, \"\\t재구성 손실:\", reconstruction_loss_val, \"\\t잠재 손실:\", latent_loss_val)  # not shown\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")  # not shown\n",
    "    \n",
    "    codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "    outputs_val = outputs.eval(feed_dict={hidden3: codings_rnd})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 62696 characters]\n",
      "text/plain": [
       "<Figure size 576x3600 with 60 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,50)) # 책에는 없음\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 10, iteration + 1)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 60312 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "plot_multiple_images(outputs_val.reshape(-1, 28, 28), n_rows, n_cols)\n",
    "save_fig(\"generated_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "여기에서 잠재 변수 손실은 조금 다르게 계산된 점을 주목하세요:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "latent_loss = 0.5 * tf.reduce_sum(\n",
    "    tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 인코드 & 디코드"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "인코드:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    }
   ],
   "source": [
    "n_digits = 3\n",
    "X_test, y_test = mnist.test.next_batch(batch_size)\n",
    "codings = hidden3\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "디코드:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_val})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "재구성 이미지를 그려봅니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9100 characters]\n",
      "text/plain": [
       "<Figure size 576x540 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8, 2.5 * n_digits))\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 2, 1 + 2 * iteration)\n",
    "    plot_image(X_test[iteration])\n",
    "    plt.subplot(n_digits, 2, 2 + 2 * iteration)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 숫자 이미지 보간"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 8428 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 9016 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 8932 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 8444 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_iterations = 3\n",
    "n_digits = 6\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    target_codings = np.roll(codings_rnd, -1, axis=0)\n",
    "    for iteration in range(n_iterations + 1):\n",
    "        codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n",
    "        outputs_val = outputs.eval(feed_dict={codings: codings_interpolate})\n",
    "        plt.figure(figsize=(11, 1.5*n_iterations))\n",
    "        for digit_index in range(n_digits):\n",
    "            plt.subplot(1, n_digits, digit_index + 1)\n",
    "            plot_image(outputs_val[digit_index])\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 해답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coming soon..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "nav_menu": {
   "height": "381px",
   "width": "453px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
