Format: application/octet-stream
1 {
2 "nbformat": 4,
3 "nbformat_minor": 0,
4 "metadata": {
5 "accelerator": "GPU",
6 "colab": {
7 "name": "NLP Course - Week 3 Exercise Answer.ipynb",
8 "provenance": [],
9 "toc_visible": true
10 },
11 "kernelspec": {
12 "display_name": "Python 3",
13 "language": "python",
14 "name": "python3"
15 },
16 "language_info": {
17 "codemirror_mode": {
18 "name": "ipython",
19 "version": 3
20 },
21 "file_extension": ".py",
22 "mimetype": "text/x-python",
23 "name": "python",
24 "nbconvert_exporter": "python",
25 "pygments_lexer": "ipython3",
26 "version": "3.7.6"
27 }
28 },
29 "cells": [
30 {
31 "cell_type": "code",
32 "metadata": {
33 "id": "zX4Kg8DUTKWO"
34 },
35 "source": [
36 "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
37 "# you may not use this file except in compliance with the License.\n",
38 "# You may obtain a copy of the License at\n",
39 "#\n",
40 "# https://www.apache.org/licenses/LICENSE-2.0\n",
41 "#\n",
42 "# Unless required by applicable law or agreed to in writing, software\n",
43 "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
44 "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
45 "# See the License for the specific language governing permissions and\n",
46 "# limitations under the License."
47 ],
48 "execution_count": null,
49 "outputs": []
50 },
51 {
52 "cell_type": "markdown",
53 "metadata": {
54 "id": "gbLDyB6LY32p"
55 },
56 "source": [
57 "<a href=\"https://colab.research.google.com/github/lmoroney/dlaicourse/blob/master/TensorFlow%20In%20Practice/Course%203%20-%20NLP/NLP%20Course%20-%20Week%203%20Exercise%20Answer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
58 ]
59 },
60 {
61 "cell_type": "code",
62 "metadata": {
63 "id": "hmA6EzkQJ5jt"
64 },
65 "source": [
66 "import json\n",
67 "import tensorflow as tf\n",
68 "import csv\n",
69 "import random\n",
70 "import numpy as np\n",
71 "\n",
72 "from tensorflow.keras.preprocessing.text import Tokenizer\n",
73 "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
74 "from tensorflow.keras.utils import to_categorical\n",
75 "from tensorflow.keras import regularizers\n",
76 "\n",
77 "\n",
78 "embedding_dim = 100\n",
79 "max_length = 16\n",
80 "trunc_type='post'\n",
81 "padding_type='post'\n",
82 "oov_tok = \"<OOV>\"\n",
83 "training_size=160000\n",
84 "test_portion=.1\n",
85 "\n",
86 "corpus = []\n"
87 ],
88 "execution_count": null,
89 "outputs": []
90 },
91 {
92 "cell_type": "code",
93 "metadata": {
94 "colab": {
95 "base_uri": "https://localhost:8080/",
96 "height": 204
97 },
98 "id": "bM0l_dORKqE0",
99 "outputId": "491ba86b-f780-4355-a4be-765565a29c8c"
100 },
101 "source": [
102 "# Note that I cleaned the Stanford dataset to remove LATIN1 encoding to make it easier for Python CSV reader\n",
103 "# You can do that yourself with:\n",
104 "# iconv -f LATIN1 -t UTF8 training.1600000.processed.noemoticon.csv -o training_cleaned.csv\n",
105 "# I then hosted it on my site to make it easier to use in this notebook\n",
106 "\n",
107 "!wget --no-check-certificate \\\n",
108 " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/training_cleaned.csv \\\n",
109 " -O /tmp/training_cleaned.csv\n",
110 "\n",
111 "num_sentences = 0\n",
112 "\n",
113 "with open(\"/tmp/training_cleaned.csv\") as csvfile:\n",
114 " reader = csv.reader(csvfile, delimiter=',')\n",
115 " for row in reader:\n",
116 " list_item=[]\n",
117 " list_item.append(row[5])\n",
118 " this_label=row[0]\n",
119 " if this_label=='0':\n",
120 " list_item.append(0)\n",
121 " else:\n",
122 " list_item.append(1)\n",
123 " num_sentences = num_sentences + 1\n",
124 " corpus.append(list_item)\n"
125 ],
126 "execution_count": null,
127 "outputs": [
128 {
129 "output_type": "stream",
130 "text": [
131 "--2019-06-07 17:53:35-- https://storage.googleapis.com/laurencemoroney-blog.appspot.com/training_cleaned.csv\n",
132 "Resolving storage.googleapis.com (storage.googleapis.com)... 173.194.192.128, 2607:f8b0:4001:c1d::80\n",
133 "Connecting to storage.googleapis.com (storage.googleapis.com)|173.194.192.128|:443... connected.\n",
134 "HTTP request sent, awaiting response... 200 OK\n",
135 "Length: 238942690 (228M) [application/octet-stream]\n",
136 "Saving to: ‘/tmp/training_cleaned.csv’\n",
137 "\n",
138 "/tmp/training_clean 100%[===================>] 227.87M 221MB/s in 1.0s \n",
139 "\n",
140 "2019-06-07 17:53:36 (221 MB/s) - ‘/tmp/training_cleaned.csv’ saved [238942690/238942690]\n",
141 "\n"
142 ],
143 "name": "stdout"
144 }
145 ]
146 },
147 {
148 "cell_type": "code",
149 "metadata": {
150 "colab": {
151 "base_uri": "https://localhost:8080/",
152 "height": 68
153 },
154 "id": "3kxblBUjEUX-",
155 "outputId": "3c0227a2-e74b-4d9b-cabb-f9ee150571b1"
156 },
157 "source": [
158 "print(num_sentences)\n",
159 "print(len(corpus))\n",
160 "print(corpus[1])\n",
161 "\n",
162 "# Expected Output:\n",
163 "# 1600000\n",
164 "# 1600000\n",
165 "# [\"is upset that he can't update his Facebook by texting it... and might cry as a result School today also. Blah!\", 0]"
166 ],
167 "execution_count": null,
168 "outputs": [
169 {
170 "output_type": "stream",
171 "text": [
172 "1600000\n",
173 "1600000\n",
174 "[\"is upset that he can't update his Facebook by texting it... and might cry as a result School today also. Blah!\", 0]\n"
175 ],
176 "name": "stdout"
177 }
178 ]
179 },
180 {
181 "cell_type": "code",
182 "metadata": {
183 "id": "ohOGz24lsNAD"
184 },
185 "source": [
186 "sentences=[]\n",
187 "labels=[]\n",
188 "random.shuffle(corpus)\n",
189 "for x in range(training_size):\n",
190 " sentences.append(corpus[x][0])\n",
191 " labels.append(corpus[x][1])\n",
192 "\n",
193 "\n",
194 "tokenizer = Tokenizer()\n",
195 "tokenizer.fit_on_texts(sentences)\n",
196 "\n",
197 "word_index = tokenizer.word_index\n",
198 "vocab_size=len(word_index)\n",
199 "\n",
200 "sequences = tokenizer.texts_to_sequences(sentences)\n",
201 "padded = pad_sequences(sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n",
202 "\n",
203 "split = int(test_portion * training_size)\n",
204 "\n",
205 "test_sequences = padded[0:split]\n",
206 "training_sequences = padded[split:training_size]\n",
207 "test_labels = labels[0:split]\n",
208 "training_labels = labels[split:training_size]"
209 ],
210 "execution_count": null,
211 "outputs": []
212 },
213 {
214 "cell_type": "code",
215 "metadata": {
216 "colab": {
217 "base_uri": "https://localhost:8080/",
218 "height": 51
219 },
220 "id": "gIrtRem1En3N",
221 "outputId": "4ad8401c-8dba-420d-8aee-38dac0b0839a"
222 },
223 "source": [
224 "print(vocab_size)\n",
225 "print(word_index['i'])\n",
226 "# Expected Output\n",
227 "# 138858\n",
228 "# 1"
229 ],
230 "execution_count": null,
231 "outputs": [
232 {
233 "output_type": "stream",
234 "text": [
235 "138858\n",
236 "1\n"
237 ],
238 "name": "stdout"
239 }
240 ]
241 },
242 {
243 "cell_type": "code",
244 "metadata": {
245 "colab": {
246 "base_uri": "https://localhost:8080/",
247 "height": 204
248 },
249 "id": "C1zdgJkusRh0",
250 "outputId": "b6edd322-8191-45e7-cb12-08921685a72f"
251 },
252 "source": [
253 "# Note this is the 100 dimension version of GloVe from Stanford\n",
254 "# I unzipped and hosted it on my site to make this notebook easier\n",
255 "!wget --no-check-certificate \\\n",
256 " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \\\n",
257 " -O /tmp/glove.6B.100d.txt\n",
258 "embeddings_index = {};\n",
259 "with open('/tmp/glove.6B.100d.txt') as f:\n",
260 " for line in f:\n",
261 " values = line.split();\n",
262 " word = values[0];\n",
263 " coefs = np.asarray(values[1:], dtype='float32');\n",
264 " embeddings_index[word] = coefs;\n",
265 "\n",
266 "embeddings_matrix = np.zeros((vocab_size+1, embedding_dim));\n",
267 "for word, i in word_index.items():\n",
268 " embedding_vector = embeddings_index.get(word);\n",
269 " if embedding_vector is not None:\n",
270 " embeddings_matrix[i] = embedding_vector;"
271 ],
272 "execution_count": null,
273 "outputs": [
274 {
275 "output_type": "stream",
276 "text": [
277 "--2019-06-07 17:55:30-- https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt\n",
278 "Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.183.128, 2607:f8b0:4001:c12::80\n",
279 "Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.183.128|:443... connected.\n",
280 "HTTP request sent, awaiting response... 200 OK\n",
281 "Length: 347116733 (331M) [text/plain]\n",
282 "Saving to: ‘/tmp/glove.6B.100d.txt’\n",
283 "\n",
284 "/tmp/glove.6B.100d. 100%[===================>] 331.04M 160MB/s in 2.1s \n",
285 "\n",
286 "2019-06-07 17:55:33 (160 MB/s) - ‘/tmp/glove.6B.100d.txt’ saved [347116733/347116733]\n",
287 "\n"
288 ],
289 "name": "stdout"
290 }
291 ]
292 },
293 {
294 "cell_type": "code",
295 "metadata": {
296 "colab": {
297 "base_uri": "https://localhost:8080/",
298 "height": 34
299 },
300 "id": "71NLk_lpFLNt",
301 "outputId": "97cb88db-754f-4375-fdc3-876cd6b4fdce"
302 },
303 "source": [
304 "print(len(embeddings_matrix))\n",
305 "# Expected Output\n",
306 "# 138859"
307 ],
308 "execution_count": null,
309 "outputs": [
310 {
311 "output_type": "stream",
312 "text": [
313 "138859\n"
314 ],
315 "name": "stdout"
316 }
317 ]
318 },
319 {
320 "cell_type": "code",
321 "metadata": {
322 "id": "iKKvbuEBOGFz"
323 },
324 "source": [
325 "model = tf.keras.Sequential([\n",
326 " tf.keras.layers.Embedding(vocab_size+1, embedding_dim, input_length=max_length, weights=[embeddings_matrix], trainable=False),\n",
327 " tf.keras.layers.Dropout(0.2),\n",
328 " tf.keras.layers.Conv1D(64, 5, activation='relu'),\n",
329 " tf.keras.layers.MaxPooling1D(pool_size=4),\n",
330 " tf.keras.layers.LSTM(64),\n",
331 " tf.keras.layers.Dense(1, activation='sigmoid')\n",
332 "])\n",
333 "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
334 "model.summary()\n",
335 "\n",
336 "num_epochs = 50\n",
337 "\n",
338 "training_padded = np.array(training_sequences)\n",
339 "training_labels = np.array(training_labels)\n",
340 "testing_padded = np.array(test_sequences)\n",
341 "testing_labels = np.array(test_labels)\n",
342 "\n",
343 "history = model.fit(training_padded, training_labels, epochs=num_epochs, validation_data=(testing_padded, testing_labels), verbose=2)\n",
344 "\n",
345 "print(\"Training Complete\")"
346 ],
347 "execution_count": null,
348 "outputs": []
349 },
350 {
351 "cell_type": "code",
352 "metadata": {
353 "id": "qxju4ItJKO8F"
354 },
355 "source": [
356 "import matplotlib.image as mpimg\n",
357 "import matplotlib.pyplot as plt\n",
358 "\n",
359 "#-----------------------------------------------------------\n",
360 "# Retrieve a list of list results on training and test data\n",
361 "# sets for each training epoch\n",
362 "#-----------------------------------------------------------\n",
363 "acc=history.history['accuracy']\n",
364 "val_acc=history.history['val_accuracy']\n",
365 "loss=history.history['loss']\n",
366 "val_loss=history.history['val_loss']\n",
367 "\n",
368 "epochs=range(len(acc)) # Get number of epochs\n",
369 "\n",
370 "#------------------------------------------------\n",
371 "# Plot training and validation accuracy per epoch\n",
372 "#------------------------------------------------\n",
373 "plt.plot(epochs, acc, 'r')\n",
374 "plt.plot(epochs, val_acc, 'b')\n",
375 "plt.title('Training and validation accuracy')\n",
376 "plt.xlabel(\"Epochs\")\n",
377 "plt.ylabel(\"Accuracy\")\n",
378 "plt.legend([\"Accuracy\", \"Validation Accuracy\"])\n",
379 "\n",
380 "plt.figure()\n",
381 "\n",
382 "#------------------------------------------------\n",
383 "# Plot training and validation loss per epoch\n",
384 "#------------------------------------------------\n",
385 "plt.plot(epochs, loss, 'r')\n",
386 "plt.plot(epochs, val_loss, 'b')\n",
387 "plt.title('Training and validation loss')\n",
388 "plt.xlabel(\"Epochs\")\n",
389 "plt.ylabel(\"Loss\")\n",
390 "plt.legend([\"Loss\", \"Validation Loss\"])\n",
391 "\n",
392 "plt.figure()\n",
393 "\n",
394 "\n",
395 "# Expected Output\n",
396 "# A chart where the validation loss does not increase sharply!"
397 ],
398 "execution_count": null,
399 "outputs": []
400 }
401 ]
402 }