1  {
   2    "nbformat": 4,
   3    "nbformat_minor": 0,
   4    "metadata": {
   5      "accelerator": "GPU",
   6      "colab": {
   7        "name": "NLP_Week4_Exercise_Shakespeare_Answer.ipynb",
   8        "provenance": [],
   9        "toc_visible": true
  10      },
  11      "kernelspec": {
  12        "display_name": "Python 3",
  13        "name": "python3"
  14      }
  15    },
  16    "cells": [
  17      {
  18        "cell_type": "code",
  19        "metadata": {
  20          "id": "zX4Kg8DUTKWO"
  21        },
  22        "source": [
  23          "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
  24          "# you may not use this file except in compliance with the License.\n",
  25          "# You may obtain a copy of the License at\n",
  26          "#\n",
  27          "# https://www.apache.org/licenses/LICENSE-2.0\n",
  28          "#\n",
  29          "# Unless required by applicable law or agreed to in writing, software\n",
  30          "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
  31          "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
  32          "# See the License for the specific language governing permissions and\n",
  33          "# limitations under the License."
  34        ],
  35        "execution_count": null,
  36        "outputs": []
  37      },
  38      {
  39        "cell_type": "code",
  40        "metadata": {
  41          "id": "BOwsuGQQY9OL"
  42        },
  43        "source": [
  44          "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
  45          "from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Bidirectional\n",
  46          "from tensorflow.keras.preprocessing.text import Tokenizer\n",
  47          "from tensorflow.keras.models import Sequential\n",
  48          "from tensorflow.keras.optimizers import Adam\n",
  49          "from tensorflow.keras import regularizers\n",
  50          "import tensorflow.keras.utils as ku \n",
  51          "import numpy as np "
  52        ],
  53        "execution_count": null,
  54        "outputs": []
  55      },
  56      {
  57        "cell_type": "code",
  58        "metadata": {
  59          "id": "PRnDnCW-Z7qv"
  60        },
  61        "source": [
  62          "tokenizer = Tokenizer()\n",
  63          "!wget --no-check-certificate \\\n",
  64          "    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sonnets.txt \\\n",
  65          "    -O /tmp/sonnets.txt\n",
  66          "data = open('/tmp/sonnets.txt').read()\n",
  67          "\n",
  68          "corpus = data.lower().split(\"\\n\")\n",
  69          "\n",
  70          "\n",
  71          "tokenizer.fit_on_texts(corpus)\n",
  72          "total_words = len(tokenizer.word_index) + 1\n",
  73          "\n",
  74          "# create input sequences using list of tokens\n",
  75          "input_sequences = []\n",
  76          "for line in corpus:\n",
  77          "\ttoken_list = tokenizer.texts_to_sequences([line])[0]\n",
  78          "\tfor i in range(1, len(token_list)):\n",
  79          "\t\tn_gram_sequence = token_list[:i+1]\n",
  80          "\t\tinput_sequences.append(n_gram_sequence)\n",
  81          "\n",
  82          "\n",
  83          "# pad sequences \n",
  84          "max_sequence_len = max([len(x) for x in input_sequences])\n",
  85          "input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))\n",
  86          "\n",
  87          "# create predictors and label\n",
  88          "predictors, label = input_sequences[:,:-1],input_sequences[:,-1]\n",
  89          "\n",
  90          "label = ku.to_categorical(label, num_classes=total_words)"
  91        ],
  92        "execution_count": null,
  93        "outputs": []
  94      },
  95      {
  96        "cell_type": "code",
  97        "metadata": {
  98          "id": "w9vH8Y59ajYL"
  99        },
 100        "source": [
 101          "model = Sequential()\n",
 102          "model.add(Embedding(total_words, 100, input_length=max_sequence_len-1))\n",
 103          "model.add(Bidirectional(LSTM(150, return_sequences = True)))\n",
 104          "model.add(Dropout(0.2))\n",
 105          "model.add(LSTM(100))\n",
 106          "model.add(Dense(total_words/2, activation='relu', kernel_regularizer=regularizers.l2(0.01)))\n",
 107          "model.add(Dense(total_words, activation='softmax'))\n",
 108          "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
 109          "print(model.summary())\n"
 110        ],
 111        "execution_count": null,
 112        "outputs": []
 113      },
 114      {
 115        "cell_type": "code",
 116        "metadata": {
 117          "id": "AIg2f1HBxqof"
 118        },
 119        "source": [
 120          " history = model.fit(predictors, label, epochs=100, verbose=1)"
 121        ],
 122        "execution_count": null,
 123        "outputs": []
 124      },
 125      {
 126        "cell_type": "code",
 127        "metadata": {
 128          "id": "1fXTEO3GJ282"
 129        },
 130        "source": [
 131          "import matplotlib.pyplot as plt\n",
 132          "acc = history.history['accuracy']\n",
 133          "loss = history.history['loss']\n",
 134          "\n",
 135          "epochs = range(len(acc))\n",
 136          "\n",
 137          "plt.plot(epochs, acc, 'b', label='Training accuracy')\n",
 138          "plt.title('Training accuracy')\n",
 139          "\n",
 140          "plt.figure()\n",
 141          "\n",
 142          "plt.plot(epochs, loss, 'b', label='Training Loss')\n",
 143          "plt.title('Training loss')\n",
 144          "plt.legend()\n",
 145          "\n",
 146          "plt.show()"
 147        ],
 148        "execution_count": null,
 149        "outputs": []
 150      },
 151      {
 152        "cell_type": "code",
 153        "metadata": {
 154          "id": "6Vc6PHgxa6Hm"
 155        },
 156        "source": [
 157          "seed_text = \"Help me Obi Wan Kenobi, you're my only hope\"\n",
 158          "next_words = 100\n",
 159          "  \n",
 160          "for _ in range(next_words):\n",
 161          "\ttoken_list = tokenizer.texts_to_sequences([seed_text])[0]\n",
 162          "\ttoken_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n",
 163          "\tpredicted = model.predict_classes(token_list, verbose=0)\n",
 164          "\toutput_word = \"\"\n",
 165          "\tfor word, index in tokenizer.word_index.items():\n",
 166          "\t\tif index == predicted:\n",
 167          "\t\t\toutput_word = word\n",
 168          "\t\t\tbreak\n",
 169          "\tseed_text += \" \" + output_word\n",
 170          "print(seed_text)"
 171        ],
 172        "execution_count": null,
 173        "outputs": []
 174      }
 175    ]
 176  }