{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "accelerator": "GPU", "colab": { "name": "NLP Course - Week 3 Exercise Answer.ipynb", "provenance": [], "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.6" } }, "cells": [ { "cell_type": "code", "metadata": { "id": "zX4Kg8DUTKWO" }, "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "gbLDyB6LY32p" }, "source": [ "\"Open" ] }, { "cell_type": "code", "metadata": { "id": "hmA6EzkQJ5jt" }, "source": [ "import json\n", "import tensorflow as tf\n", "import csv\n", "import random\n", "import numpy as np\n", "\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "from tensorflow.keras.utils import to_categorical\n", "from tensorflow.keras import regularizers\n", "\n", "\n", "embedding_dim = 100\n", "max_length = 16\n", "trunc_type='post'\n", "padding_type='post'\n", "oov_tok = \"\"\n", "training_size=160000\n", "test_portion=.1\n", "\n", "corpus = []\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 204 }, "id": "bM0l_dORKqE0", "outputId": "491ba86b-f780-4355-a4be-765565a29c8c" }, "source": [ "# Note that I cleaned the Stanford dataset to remove LATIN1 encoding to make it easier for Python CSV reader\n", "# You can do that yourself with:\n", "# iconv -f LATIN1 -t UTF8 training.1600000.processed.noemoticon.csv -o training_cleaned.csv\n", "# I then hosted it on my site to make it easier to use in this notebook\n", "\n", "!wget --no-check-certificate \\\n", " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/training_cleaned.csv \\\n", " -O /tmp/training_cleaned.csv\n", "\n", "num_sentences = 0\n", "\n", "with open(\"/tmp/training_cleaned.csv\") as csvfile:\n", " reader = csv.reader(csvfile, delimiter=',')\n", " for row in reader:\n", " list_item=[]\n", " list_item.append(row[5])\n", " this_label=row[0]\n", " if this_label=='0':\n", " list_item.append(0)\n", " else:\n", " list_item.append(1)\n", " num_sentences = num_sentences + 1\n", " corpus.append(list_item)\n" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "--2019-06-07 17:53:35-- https://storage.googleapis.com/laurencemoroney-blog.appspot.com/training_cleaned.csv\n", "Resolving storage.googleapis.com (storage.googleapis.com)... 173.194.192.128, 2607:f8b0:4001:c1d::80\n", "Connecting to storage.googleapis.com (storage.googleapis.com)|173.194.192.128|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 238942690 (228M) [application/octet-stream]\n", "Saving to: ‘/tmp/training_cleaned.csv’\n", "\n", "/tmp/training_clean 100%[===================>] 227.87M 221MB/s in 1.0s \n", "\n", "2019-06-07 17:53:36 (221 MB/s) - ‘/tmp/training_cleaned.csv’ saved [238942690/238942690]\n", "\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 68 }, "id": "3kxblBUjEUX-", "outputId": "3c0227a2-e74b-4d9b-cabb-f9ee150571b1" }, "source": [ "print(num_sentences)\n", "print(len(corpus))\n", "print(corpus[1])\n", "\n", "# Expected Output:\n", "# 1600000\n", "# 1600000\n", "# [\"is upset that he can't update his Facebook by texting it... and might cry as a result School today also. Blah!\", 0]" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "1600000\n", "1600000\n", "[\"is upset that he can't update his Facebook by texting it... and might cry as a result School today also. Blah!\", 0]\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "ohOGz24lsNAD" }, "source": [ "sentences=[]\n", "labels=[]\n", "random.shuffle(corpus)\n", "for x in range(training_size):\n", " sentences.append(corpus[x][0])\n", " labels.append(corpus[x][1])\n", "\n", "\n", "tokenizer = Tokenizer()\n", "tokenizer.fit_on_texts(sentences)\n", "\n", "word_index = tokenizer.word_index\n", "vocab_size=len(word_index)\n", "\n", "sequences = tokenizer.texts_to_sequences(sentences)\n", "padded = pad_sequences(sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", "\n", "split = int(test_portion * training_size)\n", "\n", "test_sequences = padded[0:split]\n", "training_sequences = padded[split:training_size]\n", "test_labels = labels[0:split]\n", "training_labels = labels[split:training_size]" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 51 }, "id": "gIrtRem1En3N", "outputId": "4ad8401c-8dba-420d-8aee-38dac0b0839a" }, "source": [ "print(vocab_size)\n", "print(word_index['i'])\n", "# Expected Output\n", "# 138858\n", "# 1" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "138858\n", "1\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 204 }, "id": "C1zdgJkusRh0", "outputId": "b6edd322-8191-45e7-cb12-08921685a72f" }, "source": [ "# Note this is the 100 dimension version of GloVe from Stanford\n", "# I unzipped and hosted it on my site to make this notebook easier\n", "!wget --no-check-certificate \\\n", " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \\\n", " -O /tmp/glove.6B.100d.txt\n", "embeddings_index = {};\n", "with open('/tmp/glove.6B.100d.txt') as f:\n", " for line in f:\n", " values = line.split();\n", " word = values[0];\n", " coefs = np.asarray(values[1:], dtype='float32');\n", " embeddings_index[word] = coefs;\n", "\n", "embeddings_matrix = np.zeros((vocab_size+1, embedding_dim));\n", "for word, i in word_index.items():\n", " embedding_vector = embeddings_index.get(word);\n", " if embedding_vector is not None:\n", " embeddings_matrix[i] = embedding_vector;" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "--2019-06-07 17:55:30-- https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt\n", "Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.183.128, 2607:f8b0:4001:c12::80\n", "Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.183.128|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 347116733 (331M) [text/plain]\n", "Saving to: ‘/tmp/glove.6B.100d.txt’\n", "\n", "/tmp/glove.6B.100d. 100%[===================>] 331.04M 160MB/s in 2.1s \n", "\n", "2019-06-07 17:55:33 (160 MB/s) - ‘/tmp/glove.6B.100d.txt’ saved [347116733/347116733]\n", "\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "id": "71NLk_lpFLNt", "outputId": "97cb88db-754f-4375-fdc3-876cd6b4fdce" }, "source": [ "print(len(embeddings_matrix))\n", "# Expected Output\n", "# 138859" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "138859\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "iKKvbuEBOGFz" }, "source": [ "model = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size+1, embedding_dim, input_length=max_length, weights=[embeddings_matrix], trainable=False),\n", " tf.keras.layers.Dropout(0.2),\n", " tf.keras.layers.Conv1D(64, 5, activation='relu'),\n", " tf.keras.layers.MaxPooling1D(pool_size=4),\n", " tf.keras.layers.LSTM(64),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", "model.summary()\n", "\n", "num_epochs = 50\n", "\n", "training_padded = np.array(training_sequences)\n", "training_labels = np.array(training_labels)\n", "testing_padded = np.array(test_sequences)\n", "testing_labels = np.array(test_labels)\n", "\n", "history = model.fit(training_padded, training_labels, epochs=num_epochs, validation_data=(testing_padded, testing_labels), verbose=2)\n", "\n", "print(\"Training Complete\")" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "qxju4ItJKO8F" }, "source": [ "import matplotlib.image as mpimg\n", "import matplotlib.pyplot as plt\n", "\n", "#-----------------------------------------------------------\n", "# Retrieve a list of list results on training and test data\n", "# sets for each training epoch\n", "#-----------------------------------------------------------\n", "acc=history.history['accuracy']\n", "val_acc=history.history['val_accuracy']\n", "loss=history.history['loss']\n", "val_loss=history.history['val_loss']\n", "\n", "epochs=range(len(acc)) # Get number of epochs\n", "\n", "#------------------------------------------------\n", "# Plot training and validation accuracy per epoch\n", "#------------------------------------------------\n", "plt.plot(epochs, acc, 'r')\n", "plt.plot(epochs, val_acc, 'b')\n", "plt.title('Training and validation accuracy')\n", "plt.xlabel(\"Epochs\")\n", "plt.ylabel(\"Accuracy\")\n", "plt.legend([\"Accuracy\", \"Validation Accuracy\"])\n", "\n", "plt.figure()\n", "\n", "#------------------------------------------------\n", "# Plot training and validation loss per epoch\n", "#------------------------------------------------\n", "plt.plot(epochs, loss, 'r')\n", "plt.plot(epochs, val_loss, 'b')\n", "plt.title('Training and validation loss')\n", "plt.xlabel(\"Epochs\")\n", "plt.ylabel(\"Loss\")\n", "plt.legend([\"Loss\", \"Validation Loss\"])\n", "\n", "plt.figure()\n", "\n", "\n", "# Expected Output\n", "# A chart where the validation loss does not increase sharply!" ], "execution_count": null, "outputs": [] } ] }