{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.5.5\n",
      "IPython 6.3.0\n",
      "\n",
      "numpy 1.14.3\n",
      "sklearn 0.19.1\n",
      "scipy 1.0.1\n",
      "matplotlib 2.2.2\n",
      "tensorflow 1.8.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,sklearn,scipy,matplotlib,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**15장 – 오토인코더**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 15장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"autoencoders\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "28x28 흑백 이미지를 그리기 위한 유틸리티 함수:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_multiple_images(images, n_rows, n_cols, pad=2):\n",
    "    images = images - images.min()  # 최소값을 0으로 만들어 패딩이 하얗게 보이도록 합니다.\n",
    "    w,h = images.shape[1:]\n",
    "    image = np.zeros(((w+pad)*n_rows+pad, (h+pad)*n_cols+pad))\n",
    "    for y in range(n_rows):\n",
    "        for x in range(n_cols):\n",
    "            image[(y*(h+pad)+pad):(y*(h+pad)+pad+h),(x*(w+pad)+pad):(x*(w+pad)+pad+w)] = images[y*n_cols+x]\n",
    "    plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 선형 오토인코더를 사용한 PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3D 데이터셋을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy.random as rnd\n",
    "\n",
    "rnd.seed(4)\n",
    "m = 200\n",
    "w1, w2 = 0.1, 0.3\n",
    "noise = 0.1\n",
    "\n",
    "angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5\n",
    "data = np.empty((m, 3))\n",
    "data[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2\n",
    "data[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2\n",
    "data[:, 2] = data[:, 0] * w1 + data[:, 1] * w2 + noise * rnd.randn(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터를 정규화합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(data[:100])\n",
    "X_test = scaler.transform(data[100:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "오토인코더를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "reset_graph()\n",
    "\n",
    "n_inputs = 3\n",
    "n_hidden = 2  # 코딩 유닛\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden = tf.layers.dense(X, n_hidden)\n",
    "outputs = tf.layers.dense(hidden, n_outputs)\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(reconstruction_loss)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 1000\n",
    "codings = hidden\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        training_op.run(feed_dict={X: X_train})\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 6728 characters]\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(4,3))\n",
    "plt.plot(codings_val[:,0], codings_val[:, 1], \"b.\")\n",
    "plt.xlabel(\"$z_1$\", fontsize=18)\n",
    "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n",
    "save_fig(\"linear_autoencoder_pca_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 적층 오토인코더"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST 데이터셋을 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "tf.logging.set_verbosity(tf.logging.ERROR)  # deprecated 경고 메세지를 출력하지 않기 위해 \n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 한 번에 모든 층을 훈련하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3개의 은닉층과 1개의 출력층(즉, 두 개를 적층)을 가진 적층 오토인코더를 만들어 보겠습니다. ELU 활성화 함수와 He 초기화, L2 정규화를 사용하겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "he_init = tf.contrib.layers.variance_scaling_initializer() # He 초기화\n",
    "#아래와 동일합니다:\n",
    "#he_init = lambda shape, dtype=tf.float32: tf.truncated_normal(shape, 0., stddev=np.sqrt(2/shape[0]))\n",
    "l2_regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "my_dense_layer = partial(tf.layers.dense,\n",
    "                         activation=tf.nn.elu,\n",
    "                         kernel_initializer=he_init,\n",
    "                         kernel_regularizer=l2_regularizer)\n",
    "\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3 = my_dense_layer(hidden2, n_hidden3)\n",
    "outputs = my_dense_layer(hidden3, n_outputs, activation=None)\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "loss = tf.add_n([reconstruction_loss] + reg_losses)\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver() # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 훈련시켜 보죠! 여기에서는 타깃 값을 주입하지 않습니다(`y_batch`가 사용되지 않습니다). 이는 비지도 학습입니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.020550018\n",
      "1 훈련 MSE: 0.01137075\n",
      "2 훈련 MSE: 0.0102255605\n",
      "3 훈련 MSE: 0.009902374\n",
      "4 훈련 MSE: 0.010377134\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\") # 책에는 없음\n",
    "            sys.stdout.flush()                                          # 책에는 없음\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})   # 책에는 없음\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)           # 책에는 없음\n",
    "        saver.save(sess, \"./my_model_all_layers.ckpt\")                  # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 함수는 모델을 로드하고 테스트 세트에서 이를 평가합니다(재구성 오차를 측정합니다). 그런 다음 원본 이미지와 재구성 이미지를 그립니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):\n",
    "    with tf.Session() as sess:\n",
    "        if model_path:\n",
    "            saver.restore(sess, model_path)\n",
    "        X_test = mnist.test.images[:n_test_digits]\n",
    "        outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "    fig = plt.figure(figsize=(8, 3 * n_test_digits))\n",
    "    for digit_index in range(n_test_digits):\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "        plot_image(X_test[digit_index])\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "        plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_all_layers.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 10036 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_all_layers.ckpt\")\n",
    "save_fig(\"reconstruction_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 가중치 묶기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "인코더와 디코더의 가중치를 묶는 일은 자주 있습니다(`weights_decoder = tf.transpose(weights_encoder)`). 안타깝지만 `tf.layers.dense()` 함수를 사용해서 이렇게 하기는 불가능합니다(또는 매우 어렵습니다). 수동으로 직접 오토인코더를 만들어야 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # 가중치 묶기\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # 가중치 묶기\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = reconstruction_loss + reg_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.01506695\n",
      "1 훈련 MSE: 0.016488748\n",
      "2 훈련 MSE: 0.017375939\n",
      "3 훈련 MSE: 0.016878333\n",
      "4 훈련 MSE: 0.015587721\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        saver.save(sess, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_tying_weights.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 9596 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 여러 개의 그래프에서 오토인토더를 따로따로 훈련하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하나의 오토인코더를 따로따로 훈련하는 방법이 많이 있습니다. 첫 번째 방법은 각 오토인코더를 다른 그래프를 사용하여 훈련하는 것입니다. 그런 다음 이런 오토인코더의 가중치와 편향을 복사해 초깃값으로 지정해서 적층 오토인코더를 만듭니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하나의 오토인코더를 훈련하고 변환된 훈련 세트(즉, 은닉층의 출력)와 모델 파라미터를 반환하는 함수를 만들겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "def train_autoencoder(X_train, n_neurons, n_epochs, batch_size,\n",
    "                      learning_rate = 0.01, l2_reg = 0.0005, seed=42,\n",
    "                      hidden_activation=tf.nn.elu,\n",
    "                      output_activation=tf.nn.elu):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        tf.set_random_seed(seed)\n",
    "\n",
    "        n_inputs = X_train.shape[1]\n",
    "\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "        \n",
    "        my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(l2_reg))\n",
    "\n",
    "        hidden = my_dense_layer(X, n_neurons, activation=hidden_activation, name=\"hidden\")\n",
    "        outputs = my_dense_layer(hidden, n_inputs, activation=output_activation, name=\"outputs\")\n",
    "\n",
    "        reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "        reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "        loss = tf.add_n([reconstruction_loss] + reg_losses)\n",
    "\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "        init = tf.global_variables_initializer()\n",
    "\n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        init.run()\n",
    "        for epoch in range(n_epochs):\n",
    "            n_batches = len(X_train) // batch_size\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                indices = rnd.permutation(len(X_train))[:batch_size]\n",
    "                X_batch = X_train[indices]\n",
    "                sess.run(training_op, feed_dict={X: X_batch})\n",
    "            loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n",
    "        hidden_val = hidden.eval(feed_dict={X: X_train})\n",
    "        return hidden_val, params[\"hidden/kernel:0\"], params[\"hidden/bias:0\"], params[\"outputs/kernel:0\"], params[\"outputs/bias:0\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 두 개의 오토인코더를 훈련시켜 보죠. 첫 번째는 훈련 데이터를 사용하고 두 번째는 첫 번째 오토인코더의 은닉층 출력을 사용해 훈련시킵니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.01851794\n",
      "1 훈련 MSE: 0.01868272\n",
      "2 훈련 MSE: 0.018467719\n",
      "3 훈련 MSE: 0.019231746\n",
      "0 훈련 MSE: 0.004236093\n",
      "1 훈련 MSE: 0.0048326333\n",
      "2 훈련 MSE: 0.004668708\n",
      "3 훈련 MSE: 0.0044038747\n"
     ]
    }
   ],
   "source": [
    "hidden_output, W1, b1, W4, b4 = train_autoencoder(mnist.train.images, n_neurons=300, n_epochs=4, batch_size=150,\n",
    "                                                  output_activation=None)\n",
    "_, W2, b2, W3, b3 = train_autoencoder(hidden_output, n_neurons=150, n_epochs=4, batch_size=150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "마지막으로 방금전 훈련한 오토인코더의 가중치와 편향을 재사용하여 적층 오토인코더를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden1 = tf.nn.elu(tf.matmul(X, W1) + b1)\n",
    "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n",
    "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n",
    "outputs = tf.matmul(hidden3, W4) + b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9388 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 하나의 그래프에서 오토인코더를 따로따로 훈련하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하나의 그래프를 사용하는 방법도 있습니다. 이 방법은 전체 적층 오토인코더를 위한 그래프를 만들지만 각 오토인코더를 독립적으로 훈련하기 위한 연산도 추가합니다. 단계 1은 맨 아래층과 맨 윗층을 훈련하고(즉, 첫 번째 오토인코더), 단계 2는 두 개의 가운데 층을 훈련합니다(즉, 두 번째 오토인코더)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "weights4_init = initializer([n_hidden3, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "weights4 = tf.Variable(weights4_init, dtype=tf.float32, name=\"weights4\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "with tf.name_scope(\"phase1\"):\n",
    "    phase1_outputs = tf.matmul(hidden1, weights4) + biases4  # hidden2와 hidden3 통과합니다\n",
    "    phase1_reconstruction_loss = tf.reduce_mean(tf.square(phase1_outputs - X))\n",
    "    phase1_reg_loss = regularizer(weights1) + regularizer(weights4)\n",
    "    phase1_loss = phase1_reconstruction_loss + phase1_reg_loss\n",
    "    phase1_training_op = optimizer.minimize(phase1_loss)\n",
    "\n",
    "with tf.name_scope(\"phase2\"):\n",
    "    phase2_reconstruction_loss = tf.reduce_mean(tf.square(hidden3 - hidden1))\n",
    "    phase2_reg_loss = regularizer(weights2) + regularizer(weights3)\n",
    "    phase2_loss = phase2_reconstruction_loss + phase2_reg_loss\n",
    "    train_vars = [weights2, biases2, weights3, biases3]\n",
    "    phase2_training_op = optimizer.minimize(phase2_loss, var_list=train_vars) # hidden1 동결"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "훈련 단계 #1\n",
      "0 훈련 MSE: 0.007406866\n",
      "1 훈련 MSE: 0.0078288205\n",
      "2 훈련 MSE: 0.007728097\n",
      "3 훈련 MSE: 0.0074090064\n",
      "훈련 단계 #2\n",
      "09% 훈련 MSE: 0.35758996\n",
      "1 훈련 MSE: 0.00593613\n",
      "2 훈련 MSE: 0.0030089526\n",
      "3 훈련 MSE: 0.0024365915\n",
      "테스트 MSE: 0.009815396\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op]\n",
    "reconstruction_losses = [phase1_reconstruction_loss, phase2_reconstruction_loss]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"훈련 단계 #{}\".format(phase + 1))\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                sess.run(training_ops[phase], feed_dict={X: X_batch})\n",
    "            loss_train = reconstruction_losses[phase].eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "            saver.save(sess, \"./my_model_one_at_a_time.ckpt\")\n",
    "    loss_test = reconstruction_loss.eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"테스트 MSE:\", loss_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 동결 층의 출력을 캐싱하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "훈련 단계 #1\n",
      "0 훈련 MSE: 0.0075382586\n",
      "1 훈련 MSE: 0.007754701\n",
      "2 훈련 MSE: 0.007343679\n",
      "3 훈련 MSE: 0.007837775\n",
      "훈련 단계 #2\n",
      "0 훈련 MSE: 0.18597357\n",
      "1 훈련 MSE: 0.0044647465\n",
      "2 훈련 MSE: 0.002462252\n",
      "3 훈련 MSE: 0.0020415403\n",
      "테스트 MSE: 0.009790834\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op]\n",
    "reconstruction_losses = [phase1_reconstruction_loss, phase2_reconstruction_loss]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"훈련 단계 #{}\".format(phase + 1))\n",
    "        if phase == 1:\n",
    "            hidden1_cache = hidden1.eval(feed_dict={X: mnist.train.images})\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                if phase == 1:\n",
    "                    indices = rnd.permutation(mnist.train.num_examples)\n",
    "                    hidden1_batch = hidden1_cache[indices[:batch_sizes[phase]]]\n",
    "                    feed_dict = {hidden1: hidden1_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "                else:\n",
    "                    X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                    feed_dict = {X: X_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "            loss_train = reconstruction_losses[phase].eval(feed_dict=feed_dict)\n",
    "            print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "            saver.save(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    loss_test = reconstruction_loss.eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"테스트 MSE:\", loss_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 재구성 시각화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_one_at_a_time.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 7904 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_test_digits = 2\n",
    "X_test = mnist.test.images[:n_test_digits]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_one_at_a_time.ckpt\") # not shown in the book\n",
    "    outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "for digit_index in range(n_test_digits):\n",
    "    plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "    plot_image(X_test[digit_index])\n",
    "    plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "    plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 특성 시각화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_one_at_a_time.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 10836 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_one_at_a_time.ckpt\") # 책에는 없음\n",
    "    weights1_val = weights1.eval()\n",
    "\n",
    "for i in range(5):\n",
    "    plt.subplot(1, 5, i + 1)\n",
    "    plot_image(weights1_val.T[i])\n",
    "\n",
    "save_fig(\"extracted_features_plot\") # 책에는 없음\n",
    "plt.show()                          # 책에는 없음"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 비지도 사전훈련"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST 분류 문제를 위한 작은 신경망을 만들겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_outputs), name=\"biases3\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "logits = tf.matmul(hidden2, weights3) + biases3\n",
    "\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2) + regularizer(weights3)\n",
    "loss = cross_entropy + reg_loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "pretrain_saver = tf.train.Saver([weights1, weights2, biases1, biases2])\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(사전훈련 없이)평범하게 훈련시킵니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 정확도: 0.97333336 테스트 정확도: 0.9334\n",
      "1 훈련 정확도: 0.98 테스트 정확도: 0.936\n",
      "2 훈련 정확도: 0.97333336 테스트 정확도: 0.9382\n",
      "3 훈련 정확도: 0.9866667 테스트 정확도: 0.9489\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 정확도:\", accuracy_val, end=\" \")\n",
    "        saver.save(sess, \"./my_model_supervised.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"테스트 정확도:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "사전 훈련된 오토인코더의 첫 두개의 층을 재사용해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_cache_frozen.ckpt\n",
      "0 훈련 정확도: 0.9533333\t테스트 정확도: 0.9294\n",
      "1 훈련 정확도: 0.96666664\t테스트 정확도: 0.9342\n",
      "2 훈련 정확도: 0.98\t테스트 정확도: 0.9449\n",
      "3 훈련 정확도: 0.9866667\t테스트 정확도: 0.9462\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "#training_op = optimizer.minimize(loss, var_list=[weights3, biases3])  # layers 1와 2를 동결 (선택사항)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    pretrain_saver.restore(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 정확도:\", accuracy_val, end=\"\\t\")\n",
    "        saver.save(sess, \"./my_model_supervised_pretrained.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"테스트 정확도:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 적층 잡음제거 오토인코더"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "가우시안 잡음을 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise_level = 1.0\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "X_noisy = X + noise_level * tf.random_normal(tf.shape(X))\n",
    "\n",
    "hidden1 = tf.layers.dense(X_noisy, n_hidden1, activation=tf.nn.relu,\n",
    "                          name=\"hidden1\")\n",
    "hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden2\")                            # 책에는 없음\n",
    "hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden3\")                            # 책에는 없음\n",
    "outputs = tf.layers.dense(hidden3, n_outputs, name=\"outputs\")        # 책에는 없음\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X)) # MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(reconstruction_loss)\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.044004317\n",
      "1 훈련 MSE: 0.04299514\n",
      "2 훈련 MSE: 0.042179663\n",
      "3 훈련 MSE: 0.040822495\n",
      "4 훈련 MSE: 0.040334057\n",
      "5 훈련 MSE: 0.038522195\n",
      "6 훈련 MSE: 0.039496247\n",
      "7 훈련 MSE: 0.04211809\n",
      "8 훈련 MSE: 0.03993781\n",
      "9 훈련 MSE: 0.04112029\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising_gaussian.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "드롭아웃을 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # 코딩 유닛\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropout_rate = 0.3\n",
    "\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "X_drop = tf.layers.dropout(X, dropout_rate, training=training)\n",
    "\n",
    "hidden1 = tf.layers.dense(X_drop, n_hidden1, activation=tf.nn.relu,\n",
    "                          name=\"hidden1\")\n",
    "hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden2\")                            # 책에는 없음\n",
    "hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, # 책에는 없음\n",
    "                          name=\"hidden3\")                            # 책에는 없음\n",
    "outputs = tf.layers.dense(hidden3, n_outputs, name=\"outputs\")        # 책에는 없음\n",
    "\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X)) # MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(reconstruction_loss)\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.02953044\n",
      "1 훈련 MSE: 0.027892366\n",
      "2 훈련 MSE: 0.024280708\n",
      "3 훈련 MSE: 0.024674317\n",
      "4 훈련 MSE: 0.024266083\n",
      "5 훈련 MSE: 0.024863964\n",
      "6 훈련 MSE: 0.024136161\n",
      "7 훈련 MSE: 0.025188059\n",
      "8 훈련 MSE: 0.024125345\n",
      "9 훈련 MSE: 0.024291817\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, training: True})\n",
    "        loss_train = reconstruction_loss.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", loss_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising_dropout.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_stacked_denoising_dropout.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 9592 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_stacked_denoising_dropout.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 희소 오토인코더"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 21616 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = 0.1\n",
    "q = np.linspace(0.001, 0.999, 500)\n",
    "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
    "mse = (p - q)**2\n",
    "plt.plot([p, p], [0, 0.3], \"k:\")\n",
    "plt.text(0.05, 0.32, \"목표 희소\", fontsize=14)\n",
    "plt.plot(q, kl_div, \"b-\", label=\"쿨백 라이블러 발산\")\n",
    "plt.plot(q, mse, \"r--\", label=\"MSE\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"실제 희소\")\n",
    "plt.ylabel(\"비용\", rotation=0)\n",
    "plt.axis([0, 1, 0, 0.95])\n",
    "save_fig(\"sparsity_loss_plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 1000  # 희소 코딩 유닛\n",
    "n_outputs = n_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kl_divergence(p, q):\n",
    "    # 쿨백 라이블러 발산\n",
    "    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))\n",
    "\n",
    "learning_rate = 0.01\n",
    "sparsity_target = 0.1\n",
    "sparsity_weight = 0.2\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])            # 책에는 없음\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.sigmoid) # 책에는 없음\n",
    "outputs = tf.layers.dense(hidden1, n_outputs)                     # 책에는 없음\n",
    "\n",
    "hidden1_mean = tf.reduce_mean(hidden1, axis=0) # 배치 평균\n",
    "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n",
    "reconstruction_loss = tf.reduce_mean(tf.square(outputs - X)) # MSE\n",
    "loss = reconstruction_loss + sparsity_weight * sparsity_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 MSE: 0.13819474 \t희소 손실: 0.4184438 \t전체 손실: 0.2218835\n",
      "1 훈련 MSE: 0.059185907 \t희소 손실: 0.010747713 \t전체 손실: 0.06133545\n",
      "2 훈련 MSE: 0.053931113 \t희소 손실: 0.01995793 \t전체 손실: 0.0579227\n",
      "3 훈련 MSE: 0.04771372 \t희소 손실: 0.039470345 \t전체 손실: 0.05560779\n",
      "4 훈련 MSE: 0.044841796 \t희소 손실: 0.011593144 \t전체 손실: 0.047160424\n",
      "5 훈련 MSE: 0.04044193 \t희소 손실: 0.092471 \t전체 손실: 0.058936134\n",
      "6 훈련 MSE: 0.03886659 \t희소 손실: 0.04581263 \t전체 손실: 0.048029117\n",
      "7 훈련 MSE: 0.037868652 \t희소 손실: 0.07498071 \t전체 손실: 0.052864797\n",
      "8 훈련 MSE: 0.033122517 \t희소 손실: 0.020321768 \t전체 손실: 0.037186872\n",
      "9 훈련 MSE: 0.031449173 \t희소 손실: 0.095670156 \t전체 손실: 0.050583206\n",
      "10 훈련 MSE: 0.027398113 \t희소 손실: 0.06570565 \t전체 손실: 0.040539242\n",
      "11 훈련 MSE: 0.024734963 \t희소 손실: 0.08889264 \t전체 손실: 0.04251349\n",
      "12 훈련 MSE: 0.023367295 \t희소 손실: 0.05766213 \t전체 손실: 0.03489972\n",
      "13 훈련 MSE: 0.023007901 \t희소 손실: 0.06193536 \t전체 손실: 0.035394974\n",
      "14 훈련 MSE: 0.02123353 \t희소 손실: 0.025881292 \t전체 손실: 0.02640979\n",
      "15 훈련 MSE: 0.022064691 \t희소 손실: 0.48820361 \t전체 손실: 0.119705416\n",
      "16 훈련 MSE: 0.01910253 \t희소 손실: 0.03220318 \t전체 손실: 0.025543166\n",
      "17 훈련 MSE: 0.018982193 \t희소 손실: 0.1299511 \t전체 손실: 0.044972412\n",
      "18 훈련 MSE: 0.017487913 \t희소 손실: 0.035001524 \t전체 손실: 0.024488218\n",
      "19 훈련 MSE: 0.017776337 \t희소 손실: 0.11024489 \t전체 손실: 0.039825313\n",
      "20 훈련 MSE: 0.01691014 \t희소 손실: 0.030320974 \t전체 손실: 0.022974335\n",
      "21 훈련 MSE: 0.018679725 \t희소 손실: 0.36376736 \t전체 손실: 0.0914332\n",
      "22 훈련 MSE: 0.016285753 \t희소 손실: 0.055482175 \t전체 손실: 0.027382188\n",
      "23 훈련 MSE: 0.015950203 \t희소 손실: 0.07936562 \t전체 손실: 0.03182333\n",
      "24 훈련 MSE: 0.015672015 \t희소 손실: 0.14289382 \t전체 손실: 0.04425078\n",
      "25 훈련 MSE: 0.014612303 \t희소 손실: 0.13330582 \t전체 손실: 0.041273467\n",
      "26 훈련 MSE: 0.014524783 \t희소 손실: 0.10948151 \t전체 손실: 0.036421087\n",
      "27 훈련 MSE: 0.013598451 \t희소 손실: 0.07208073 \t전체 손실: 0.028014597\n",
      "28 훈련 MSE: 0.014069849 \t희소 손실: 0.1578956 \t전체 손실: 0.045648966\n",
      "29 훈련 MSE: 0.01344161 \t희소 손실: 0.14449331 \t전체 손실: 0.04234027\n",
      "30 훈련 MSE: 0.013608929 \t희소 손실: 0.18506998 \t전체 손실: 0.050622925\n",
      "31 훈련 MSE: 0.014197284 \t희소 손실: 0.06788698 \t전체 손실: 0.02777468\n",
      "32 훈련 MSE: 0.013759439 \t희소 손실: 0.057576388 \t전체 손실: 0.025274716\n",
      "33 훈련 MSE: 0.013667366 \t희소 손실: 0.11363433 \t전체 손실: 0.03639423\n",
      "34 훈련 MSE: 0.01264697 \t희소 손실: 0.13469386 \t전체 손실: 0.039585743\n",
      "35 훈련 MSE: 0.012872911 \t희소 손실: 0.1685048 \t전체 손실: 0.046573874\n",
      "36 훈련 MSE: 0.012669464 \t희소 손실: 0.09312945 \t전체 손실: 0.031295355\n",
      "37 훈련 MSE: 0.012427666 \t희소 손실: 0.10538912 \t전체 손실: 0.033505492\n",
      "38 훈련 MSE: 0.012336045 \t희소 손실: 0.19236036 \t전체 손실: 0.050808117\n",
      "39 훈련 MSE: 0.0123378765 \t희소 손실: 0.1423154 \t전체 손실: 0.04080096\n",
      "40 훈련 MSE: 0.012627057 \t희소 손실: 0.23267198 \t전체 손실: 0.059161454\n",
      "41 훈련 MSE: 0.0123833865 \t희소 손실: 0.10489211 \t전체 손실: 0.033361807\n",
      "42 훈련 MSE: 0.011781143 \t희소 손실: 0.10161278 \t전체 손실: 0.0321037\n",
      "43 훈련 MSE: 0.012043943 \t희소 손실: 0.13593857 \t전체 손실: 0.039231658\n",
      "44 훈련 MSE: 0.011620727 \t희소 손실: 0.114727125 \t전체 손실: 0.034566153\n",
      "45 훈련 MSE: 0.011668878 \t희소 손실: 0.15798113 \t전체 손실: 0.043265104\n",
      "46 훈련 MSE: 0.011761186 \t희소 손실: 0.13549805 \t전체 손실: 0.038860794\n",
      "47 훈련 MSE: 0.011549666 \t희소 손실: 0.15659894 \t전체 손실: 0.042869456\n",
      "48 훈련 MSE: 0.011674238 \t희소 손실: 0.44305113 \t전체 손실: 0.100284465\n",
      "49 훈련 MSE: 0.011562435 \t희소 손실: 0.1758345 \t전체 손실: 0.046729334\n",
      "50 훈련 MSE: 0.01136023 \t희소 손실: 0.18884605 \t전체 손실: 0.04912944\n",
      "51 훈련 MSE: 0.01153872 \t희소 손실: 0.2633665 \t전체 손실: 0.06421202\n",
      "52 훈련 MSE: 0.012044075 \t희소 손실: 0.36758316 \t전체 손실: 0.0855607\n",
      "53 훈련 MSE: 0.011250994 \t희소 손실: 0.12487886 \t전체 손실: 0.036226768\n",
      "54 훈련 MSE: 0.013878834 \t희소 손실: 0.43317184 \t전체 손실: 0.100513205\n",
      "55 훈련 MSE: 0.012782411 \t희소 손실: 0.07718266 \t전체 손실: 0.028218944\n",
      "56 훈련 MSE: 0.012197349 \t희소 손실: 0.301488 \t전체 손실: 0.072494954\n",
      "57 훈련 MSE: 0.012758316 \t희소 손실: 0.24829106 \t전체 손실: 0.062416527\n",
      "58 훈련 MSE: 0.023019526 \t희소 손실: 0.23253448 \t전체 손실: 0.06952642\n",
      "59 훈련 MSE: 0.0134694 \t희소 손실: 0.1981337 \t전체 손실: 0.053096145\n",
      "60 훈련 MSE: 0.019005049 \t희소 손실: 0.19429345 \t전체 손실: 0.057863742\n",
      "61 훈련 MSE: 0.012830879 \t희소 손실: 0.22918573 \t전체 손실: 0.058668025\n",
      "62 훈련 MSE: 0.013951195 \t희소 손실: 0.47963494 \t전체 손실: 0.10987819\n",
      "63 훈련 MSE: 0.014569022 \t희소 손실: 0.33389828 \t전체 손실: 0.08134868\n",
      "64 훈련 MSE: 0.011955362 \t희소 손실: 0.17778753 \t전체 손실: 0.047512867\n",
      "65 훈련 MSE: 0.016137443 \t희소 손실: 0.18549094 \t전체 손실: 0.05323563\n",
      "66 훈련 MSE: 0.014922719 \t희소 손실: 0.9833389 \t전체 손실: 0.2115905\n",
      "67 훈련 MSE: 0.037293304 \t희소 손실: 0.31203473 \t전체 손실: 0.09970025\n",
      "68 훈련 MSE: 0.015952839 \t희소 손실: 0.21828063 \t전체 손실: 0.059608966\n",
      "69 훈련 MSE: 0.012261292 \t희소 손실: 0.43984902 \t전체 손실: 0.100231096\n",
      "70 훈련 MSE: 0.037013255 \t희소 손실: 0.7457465 \t전체 손실: 0.18616256\n",
      "71 훈련 MSE: 0.014764387 \t희소 손실: 0.24276587 \t전체 손실: 0.06331757\n",
      "72 훈련 MSE: 0.016606566 \t희소 손실: 0.7994744 \t전체 손실: 0.17650145\n",
      "73 훈련 MSE: 0.0144267725 \t희소 손실: 0.31668615 \t전체 손실: 0.077764004\n",
      "74 훈련 MSE: 0.036295358 \t희소 손실: 0.32408017 \t전체 손실: 0.1011114\n",
      "75 훈련 MSE: 0.038099732 \t희소 손실: 0.5090959 \t전체 손실: 0.13991891\n",
      "76 훈련 MSE: 0.019446107 \t희소 손실: 0.20034213 \t전체 손실: 0.059514537\n",
      "77 훈련 MSE: 0.022708207 \t희소 손실: 0.3122869 \t전체 손실: 0.08516559\n",
      "78 훈련 MSE: 0.016776377 \t희소 손실: 0.5347481 \t전체 손실: 0.123725995\n",
      "79 훈련 MSE: 0.02007503 \t희소 손실: 0.42173243 \t전체 손실: 0.10442152\n",
      "80 훈련 MSE: 0.015990663 \t희소 손실: 0.11620571 \t전체 손실: 0.039231807\n",
      "81 훈련 MSE: 0.012873366 \t희소 손실: 0.8802524 \t전체 손실: 0.18892385\n",
      "82 훈련 MSE: 0.015848754 \t희소 손실: 2.1350327 \t전체 손실: 0.4428553\n",
      "83 훈련 MSE: 0.014213436 \t희소 손실: 0.6619952 \t전체 손실: 0.14661248\n",
      "84 훈련 MSE: 0.0148504 \t희소 손실: 0.27111006 \t전체 손실: 0.06907241\n",
      "85 훈련 MSE: 0.028653098 \t희소 손실: 0.9375648 \t전체 손실: 0.21616606\n",
      "86 훈련 MSE: 0.014789838 \t희소 손실: 0.68491566 \t전체 손실: 0.15177298\n",
      "87 훈련 MSE: 0.012122109 \t희소 손실: 1.4151382 \t전체 손실: 0.29514974\n",
      "88 훈련 MSE: 0.019227153 \t희소 손실: 0.85921484 \t전체 손실: 0.19107012\n",
      "89 훈련 MSE: 0.028509108 \t희소 손실: 0.29490086 \t전체 손실: 0.087489285\n",
      "90 훈련 MSE: 0.028761363 \t희소 손실: 0.45986366 \t전체 손실: 0.120734096\n",
      "91 훈련 MSE: 0.014937726 \t희소 손실: 0.098271474 \t전체 손실: 0.03459202\n",
      "92 훈련 MSE: 0.014802266 \t희소 손실: 0.3885908 \t전체 손실: 0.09252043\n",
      "93 훈련 MSE: 0.015891917 \t희소 손실: 0.1618818 \t전체 손실: 0.048268277\n",
      "94 훈련 MSE: 0.012146741 \t희소 손실: 0.115840726 \t전체 손실: 0.035314888\n",
      "95 훈련 MSE: 0.013941603 \t희소 손실: 0.10216563 \t전체 손실: 0.03437473\n",
      "96 훈련 MSE: 0.019508341 \t희소 손실: 0.23841716 \t전체 손실: 0.06719177\n",
      "97 훈련 MSE: 0.013997071 \t희소 손실: 0.108866125 \t전체 손실: 0.035770297\n",
      "98 훈련 MSE: 0.013801815 \t희소 손실: 0.09045001 \t전체 손실: 0.031891815\n",
      "99 훈련 MSE: 0.02559923 \t희소 손실: 0.33294424 \t전체 손실: 0.092188075\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 1000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        reconstruction_loss_val, sparsity_loss_val, loss_val = sess.run([reconstruction_loss, sparsity_loss, loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 MSE:\", reconstruction_loss_val, \"\\t희소 손실:\", sparsity_loss_val, \"\\t전체 손실:\", loss_val)\n",
    "        saver.save(sess, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_sparse.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 10196 characters]\n",
      "text/plain": [
       "<Figure size 576x432 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "코딩층은 0에서 1사이의 값을 출력해야 하므로 시그모이드 활성화 함수를 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.sigmoid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훈련 속도를 높이기 위해 입력을 0과 1사이로 정규화하고 비용 함수로 MSE 대신 크로스엔트로피를 사용합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = tf.layers.dense(hidden1, n_outputs)\n",
    "outputs = tf.nn.sigmoid(logits)\n",
    "\n",
    "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits)\n",
    "reconstruction_loss = tf.reduce_mean(xentropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 변이형 오토인코더"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # 코딩 유닛\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "learning_rate = 0.001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "my_dense_layer = partial(\n",
    "    tf.layers.dense,\n",
    "    activation=tf.nn.elu,\n",
    "    kernel_initializer=initializer)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3_mean = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "hidden3_sigma = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "noise = tf.random_normal(tf.shape(hidden3_sigma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + hidden3_sigma * noise\n",
    "hidden4 = my_dense_layer(hidden3, n_hidden4)\n",
    "hidden5 = my_dense_layer(hidden4, n_hidden5)\n",
    "logits = my_dense_layer(hidden5, n_outputs, activation=None)\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits)\n",
    "reconstruction_loss = tf.reduce_sum(xentropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps = 1e-10 # NaN을 반환하는 log(0)을 피하기 위한 안전항\n",
    "latent_loss = 0.5 * tf.reduce_sum(\n",
    "    tf.square(hidden3_sigma) + tf.square(hidden3_mean)\n",
    "    - 1 - tf.log(eps + tf.square(hidden3_sigma)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 전체 손실: 38016.934 \t재구성 손실: 26717.334 \t잠재 손실: 11299.599\n",
      "1 훈련 전체 손실: 30065.5 \t재구성 손실: 24319.137 \t잠재 손실: 5746.3633\n",
      "2 훈련 전체 손실: 32188.535 \t재구성 손실: 24696.451 \t잠재 손실: 7492.085\n",
      "3 훈련 전체 손실: 29995.074 \t재구성 손실: 24591.836 \t잠재 손실: 5403.2393\n",
      "4 훈련 전체 손실: 22977.074 \t재구성 손실: 19545.846 \t잠재 손실: 3431.228\n",
      "5 훈련 전체 손실: 21461.336 \t재구성 손실: 18221.758 \t잠재 손실: 3239.5786\n",
      "6 훈련 전체 손실: 17799.531 \t재구성 손실: 14987.603 \t잠재 손실: 2811.9297\n",
      "7 훈련 전체 손실: 17872.271 \t재구성 손실: 14900.03 \t잠재 손실: 2972.2415\n",
      "8 훈련 전체 손실: 16473.062 \t재구성 손실: 13437.503 \t잠재 손실: 3035.5598\n",
      "9 훈련 전체 손실: 18285.943 \t재구성 손실: 15230.208 \t잠재 손실: 3055.7349\n",
      "10 훈련 전체 손실: 17281.297 \t재구성 손실: 14238.846 \t잠재 손실: 3042.4512\n",
      "11 훈련 전체 손실: 16178.416 \t재구성 손실: 13091.535 \t잠재 손실: 3086.881\n",
      "12 훈련 전체 손실: 16544.488 \t재구성 손실: 13217.236 \t잠재 손실: 3327.253\n",
      "13 훈련 전체 손실: 16427.021 \t재구성 손실: 13077.631 \t잠재 손실: 3349.391\n",
      "14 훈련 전체 손실: 16616.738 \t재구성 손실: 13293.669 \t잠재 손실: 3323.07\n",
      "15 훈련 전체 손실: 16271.476 \t재구성 손실: 13142.443 \t잠재 손실: 3129.0325\n",
      "16 훈련 전체 손실: 16567.523 \t재구성 손실: 13333.615 \t잠재 손실: 3233.9075\n",
      "17 훈련 전체 손실: 29101.3 \t재구성 손실: 24271.592 \t잠재 손실: 4829.709\n",
      "18 훈련 전체 손실: 24227.15 \t재구성 손실: 20683.895 \t잠재 손실: 3543.2554\n",
      "19 훈련 전체 손실: 20956.738 \t재구성 손실: 17900.092 \t잠재 손실: 3056.6475\n",
      "20 훈련 전체 손실: 20429.688 \t재구성 손실: 16620.959 \t잠재 손실: 3808.7285\n",
      "21 훈련 전체 손실: 17099.916 \t재구성 손실: 13952.262 \t잠재 손실: 3147.6538\n",
      "22 훈련 전체 손실: 16274.533 \t재구성 손실: 12991.28 \t잠재 손실: 3283.2524\n",
      "23 훈련 전체 손실: 16308.104 \t재구성 손실: 12951.634 \t잠재 손실: 3356.4697\n",
      "24 훈련 전체 손실: 15702.99 \t재구성 손실: 12309.805 \t잠재 손실: 3393.1858\n",
      "25 훈련 전체 손실: 15554.395 \t재구성 손실: 12204.829 \t잠재 손실: 3349.565\n",
      "26 훈련 전체 손실: 16080.304 \t재구성 손실: 12602.002 \t잠재 손실: 3478.3018\n",
      "27 훈련 전체 손실: 16153.012 \t재구성 손실: 12719.799 \t잠재 손실: 3433.213\n",
      "28 훈련 전체 손실: 16836.662 \t재구성 손실: 13554.496 \t잠재 손실: 3282.1663\n",
      "29 훈련 전체 손실: 16081.364 \t재구성 손실: 12673.35 \t잠재 손실: 3408.0144\n",
      "30 훈련 전체 손실: 18377.371 \t재구성 손실: 14856.055 \t잠재 손실: 3521.316\n",
      "31 훈련 전체 손실: 15566.567 \t재구성 손실: 12113.064 \t잠재 손실: 3453.503\n",
      "32 훈련 전체 손실: 15363.791 \t재구성 손실: 11939.522 \t잠재 손실: 3424.268\n",
      "33 훈련 전체 손실: 15380.105 \t재구성 손실: 11933.108 \t잠재 손실: 3446.9973\n",
      "34 훈련 전체 손실: 15197.85 \t재구성 손실: 11855.855 \t잠재 손실: 3341.9941\n",
      "35 훈련 전체 손실: 15376.002 \t재구성 손실: 11782.05 \t잠재 손실: 3593.952\n",
      "36 훈련 전체 손실: 15606.907 \t재구성 손실: 12056.068 \t잠재 손실: 3550.839\n",
      "37 훈련 전체 손실: 18351.05 \t재구성 손실: 14509.541 \t잠재 손실: 3841.5088\n",
      "38 훈련 전체 손실: 16268.131 \t재구성 손실: 12896.816 \t잠재 손실: 3371.3145\n",
      "39 훈련 전체 손실: 17762.117 \t재구성 손실: 14202.475 \t잠재 손실: 3559.6433\n",
      "40 훈련 전체 손실: 23485.436 \t재구성 손실: 19128.334 \t잠재 손실: 4357.101\n",
      "41 훈련 전체 손실: 36509.016 \t재구성 손실: 26140.258 \t잠재 손실: 10368.758\n",
      "42 훈련 전체 손실: 25619.293 \t재구성 손실: 20840.367 \t잠재 손실: 4778.926\n",
      "43 훈련 전체 손실: 28378.17 \t재구성 손실: 22257.338 \t잠재 손실: 6120.8325\n",
      "44 훈련 전체 손실: 20361.963 \t재구성 손실: 16991.43 \t잠재 손실: 3370.5337\n",
      "45 훈련 전체 손실: 18422.102 \t재구성 손실: 14912.807 \t잠재 손실: 3509.295\n",
      "46 훈련 전체 손실: 16764.645 \t재구성 손실: 13529.869 \t잠재 손실: 3234.7764\n",
      "47 훈련 전체 손실: 15870.282 \t재구성 손실: 12479.645 \t잠재 손실: 3390.6377\n",
      "48 훈련 전체 손실: 15523.098 \t재구성 손실: 12218.592 \t잠재 손실: 3304.506\n",
      "49 훈련 전체 손실: 15175.317 \t재구성 손실: 11773.2705 \t잠재 손실: 3402.0469\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_val, reconstruction_loss_val, latent_loss_val = sess.run([loss, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 전체 손실:\", loss_val, \"\\t재구성 손실:\", reconstruction_loss_val, \"\\t잠재 손실:\", latent_loss_val)\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # 코딩 유닛\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "learning_rate = 0.001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "my_dense_layer = partial(\n",
    "    tf.layers.dense,\n",
    "    activation=tf.nn.elu,\n",
    "    kernel_initializer=initializer)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3_mean = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "hidden3_gamma = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "noise = tf.random_normal(tf.shape(hidden3_gamma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + tf.exp(0.5 * hidden3_gamma) * noise\n",
    "hidden4 = my_dense_layer(hidden3, n_hidden4)\n",
    "hidden5 = my_dense_layer(hidden4, n_hidden5)\n",
    "logits = my_dense_layer(hidden5, n_outputs, activation=None)\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits)\n",
    "reconstruction_loss = tf.reduce_sum(xentropy)\n",
    "latent_loss = 0.5 * tf.reduce_sum(\n",
    "    tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)\n",
    "loss = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 숫자 이미지 생성"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "모델을 훈련시켜 랜덤한 이미지를 생성해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 훈련 전체 손실: 17792.723 \t재구성 손실: 14123.082 \t잠재 손실: 3669.6401\n",
      "1 훈련 전체 손실: 17332.346 \t재구성 손실: 13560.099 \t잠재 손실: 3772.2463\n",
      "2 훈련 전체 손실: 16350.873 \t재구성 손실: 12579.393 \t잠재 손실: 3771.4807\n",
      "3 훈련 전체 손실: 16581.45 \t재구성 손실: 12810.662 \t잠재 손실: 3770.7861\n",
      "4 훈련 전체 손실: 16224.016 \t재구성 손실: 12450.172 \t잠재 손실: 3773.8433\n",
      "5 훈련 전체 손실: 15628.262 \t재구성 손실: 11819.755 \t잠재 손실: 3808.5068\n",
      "6 훈련 전체 손실: 16081.008 \t재구성 손실: 12179.777 \t잠재 손실: 3901.231\n",
      "7 훈련 전체 손실: 15772.933 \t재구성 손실: 12021.411 \t잠재 손실: 3751.5212\n",
      "8 훈련 전체 손실: 16276.584 \t재구성 손실: 12404.785 \t잠재 손실: 3871.7988\n",
      "9 훈련 전체 손실: 15589.723 \t재구성 손실: 11740.717 \t잠재 손실: 3849.0063\n",
      "10 훈련 전체 손실: 15931.363 \t재구성 손실: 12031.385 \t잠재 손실: 3899.9783\n",
      "11 훈련 전체 손실: 16112.865 \t재구성 손실: 12238.496 \t잠재 손실: 3874.369\n",
      "12 훈련 전체 손실: 16002.043 \t재구성 손실: 12185.183 \t잠재 손실: 3816.8608\n",
      "13 훈련 전체 손실: 15357.919 \t재구성 손실: 11667.58 \t잠재 손실: 3690.3389\n",
      "14 훈련 전체 손실: 16208.517 \t재구성 손실: 12264.615 \t잠재 손실: 3943.9011\n",
      "15 훈련 전체 손실: 15970.236 \t재구성 손실: 12158.714 \t잠재 손실: 3811.522\n",
      "16 훈련 전체 손실: 15551.721 \t재구성 손실: 11783.23 \t잠재 손실: 3768.4897\n",
      "17 훈련 전체 손실: 15330.088 \t재구성 손실: 11555.771 \t잠재 손실: 3774.3167\n",
      "18 훈련 전체 손실: 15251.499 \t재구성 손실: 11584.66 \t잠재 손실: 3666.8386\n",
      "19 훈련 전체 손실: 15196.1455 \t재구성 손실: 11516.703 \t잠재 손실: 3679.4424\n",
      "20 훈련 전체 손실: 15324.054 \t재구성 손실: 11526.015 \t잠재 손실: 3798.0388\n",
      "21 훈련 전체 손실: 15358.613 \t재구성 손실: 11515.41 \t잠재 손실: 3843.2036\n",
      "22 훈련 전체 손실: 15298.06 \t재구성 손실: 11582.691 \t잠재 손실: 3715.3684\n",
      "23 훈련 전체 손실: 14673.169 \t재구성 손실: 10940.84 \t잠재 손실: 3732.329\n",
      "24 훈련 전체 손실: 15293.574 \t재구성 손실: 11561.805 \t잠재 손실: 3731.7695\n",
      "25 훈련 전체 손실: 15256.209 \t재구성 손실: 11540.664 \t잠재 손실: 3715.545\n",
      "26 훈련 전체 손실: 15305.479 \t재구성 손실: 11475.454 \t잠재 손실: 3830.025\n",
      "27 훈련 전체 손실: 15276.918 \t재구성 손실: 11449.658 \t잠재 손실: 3827.2593\n",
      "28 훈련 전체 손실: 14980.705 \t재구성 손실: 11318.064 \t잠재 손실: 3662.6406\n",
      "29 훈련 전체 손실: 15232.898 \t재구성 손실: 11520.201 \t잠재 손실: 3712.6968\n",
      "30 훈련 전체 손실: 14872.446 \t재구성 손실: 11173.004 \t잠재 손실: 3699.4421\n",
      "31 훈련 전체 손실: 14890.485 \t재구성 손실: 11144.245 \t잠재 손실: 3746.2402\n",
      "32 훈련 전체 손실: 15246.843 \t재구성 손실: 11439.51 \t잠재 손실: 3807.333\n",
      "33 훈련 전체 손실: 15063.674 \t재구성 손실: 11282.296 \t잠재 손실: 3781.3784\n",
      "34 훈련 전체 손실: 15046.693 \t재구성 손실: 11310.135 \t잠재 손실: 3736.5586\n",
      "35 훈련 전체 손실: 15293.908 \t재구성 손실: 11599.437 \t잠재 손실: 3694.4717\n",
      "36 훈련 전체 손실: 15134.561 \t재구성 손실: 11362.827 \t잠재 손실: 3771.734\n",
      "37 훈련 전체 손실: 14705.802 \t재구성 손실: 11054.809 \t잠재 손실: 3650.993\n",
      "38 훈련 전체 손실: 14914.01 \t재구성 손실: 11077.051 \t잠재 손실: 3836.9585\n",
      "39 훈련 전체 손실: 14848.272 \t재구성 손실: 11198.791 \t잠재 손실: 3649.4814\n",
      "40 훈련 전체 손실: 14694.52 \t재구성 손실: 10991.713 \t잠재 손실: 3702.807\n",
      "41 훈련 전체 손실: 15223.752 \t재구성 손실: 11464.886 \t잠재 손실: 3758.8667\n",
      "42 훈련 전체 손실: 14585.404 \t재구성 손실: 11019.355 \t잠재 손실: 3566.0483\n",
      "43 훈련 전체 손실: 14579.119 \t재구성 손실: 10931.234 \t잠재 손실: 3647.8848\n",
      "44 훈련 전체 손실: 15049.283 \t재구성 손실: 11381.96 \t잠재 손실: 3667.3232\n",
      "45 훈련 전체 손실: 14855.669 \t재구성 손실: 11125.52 \t잠재 손실: 3730.1494\n",
      "46 훈련 전체 손실: 14777.61 \t재구성 손실: 11093.045 \t잠재 손실: 3684.5652\n",
      "47 훈련 전체 손실: 14408.994 \t재구성 손실: 10788.754 \t잠재 손실: 3620.24\n",
      "48 훈련 전체 손실: 14479.196 \t재구성 손실: 10864.229 \t잠재 손실: 3614.9678\n",
      "49 훈련 전체 손실: 14638.092 \t재구성 손실: 10926.594 \t잠재 손실: 3711.4983\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "n_digits = 60\n",
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\") # not shown in the book\n",
    "            sys.stdout.flush()                                          # not shown\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        loss_val, reconstruction_loss_val, latent_loss_val = sess.run([loss, reconstruction_loss, latent_loss], feed_dict={X: X_batch}) # not shown\n",
    "        print(\"\\r{}\".format(epoch), \"훈련 전체 손실:\", loss_val, \"\\t재구성 손실:\", reconstruction_loss_val, \"\\t잠재 손실:\", latent_loss_val)  # not shown\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")  # not shown\n",
    "    \n",
    "    codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "    outputs_val = outputs.eval(feed_dict={hidden3: codings_rnd})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 63276 characters]\n",
      "text/plain": [
       "<Figure size 576x3600 with 60 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,50)) # 책에는 없음\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 10, iteration + 1)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 61256 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "plot_multiple_images(outputs_val.reshape(-1, 28, 28), n_rows, n_cols)\n",
    "save_fig(\"generated_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "여기에서 잠재 변수 손실은 조금 다르게 계산된 점을 주목하세요:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "latent_loss = 0.5 * tf.reduce_sum(\n",
    "    tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 인코드 & 디코드"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "인코드:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    }
   ],
   "source": [
    "n_digits = 3\n",
    "X_test, y_test = mnist.test.next_batch(batch_size)\n",
    "codings = hidden3\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "디코드:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_val})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "재구성 이미지를 그려봅니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8880 characters]\n",
      "text/plain": [
       "<Figure size 576x540 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8, 2.5 * n_digits))\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 2, 1 + 2 * iteration)\n",
    "    plot_image(X_test[iteration])\n",
    "    plt.subplot(n_digits, 2, 2 + 2 * iteration)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 숫자 이미지 보간"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 7524 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 7372 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 7244 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 7512 characters]\n",
      "text/plain": [
       "<Figure size 792x324 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_iterations = 3\n",
    "n_digits = 6\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    target_codings = np.roll(codings_rnd, -1, axis=0)\n",
    "    for iteration in range(n_iterations + 1):\n",
    "        codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n",
    "        outputs_val = outputs.eval(feed_dict={codings: codings_interpolate})\n",
    "        plt.figure(figsize=(11, 1.5*n_iterations))\n",
    "        for digit_index in range(n_digits):\n",
    "            plt.subplot(1, n_digits, digit_index + 1)\n",
    "            plot_image(outputs_val[digit_index])\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 연습문제 해답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coming soon..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "nav_menu": {
   "height": "381px",
   "width": "453px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
