{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b274e535",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "570d8b51",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences=['I love everything',\n",
    "            'i love nothing',\n",
    "              'Everyone love nothing and everything!',\n",
    "              'who loves who and why?']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1da9f851",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b5970c09",
   "metadata": {},
   "outputs": [],
   "source": [
    "toke= Tokenizer(oov_token='<OOV>') #out of vocabulary\n",
    "toke.fit_on_texts(sentences)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "41640cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_dict=toke.word_index\n",
    "seq= toke.texts_to_sequences(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15d0d0c4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "372f508f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3, 2, 4], [3, 2, 5], [8, 2, 5, 6, 4], [7, 9, 7, 6, 10]]\n",
      "{'<OOV>': 1, 'love': 2, 'i': 3, 'everything': 4, 'nothing': 5, 'and': 6, 'who': 7, 'everyone': 8, 'loves': 9, 'why': 10}\n"
     ]
    }
   ],
   "source": [
    "print(seq)\n",
    "print(word_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "185e2b90",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a9de96d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "padded= pad_sequences(seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5dae714f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  3  2  4]\n",
      " [ 0  0  3  2  5]\n",
      " [ 8  2  5  6  4]\n",
      " [ 7  9  7  6 10]]\n"
     ]
    }
   ],
   "source": [
    "print(padded, p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4106923b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fe96ad2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21842f8e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a60a4c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba65d624",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "970d964c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Word Index =  {'<OOV>': 1, 'my': 2, 'love': 3, 'dog': 4, 'i': 5, 'you': 6, 'cat': 7, 'do': 8, 'think': 9, 'is': 10, 'amazing': 11}\n",
      "\n",
      "Sequences =  [[5, 3, 2, 4], [5, 3, 2, 7], [6, 3, 2, 4], [8, 6, 9, 2, 4, 10, 11]]\n",
      "\n",
      "Padded Sequences:\n",
      "[[ 0  5  3  2  4]\n",
      " [ 0  5  3  2  7]\n",
      " [ 0  6  3  2  4]\n",
      " [ 9  2  4 10 11]]\n",
      "\n",
      "Test Sequence =  [[5, 1, 3, 2, 4], [2, 4, 1, 2, 1]]\n",
      "\n",
      "Padded Test Sequence: \n",
      "[[0 0 0 0 0 5 1 3 2 4]\n",
      " [0 0 0 0 0 2 4 1 2 1]]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "sentences = [\n",
    "    'I love my dog',\n",
    "    'I love my cat',\n",
    "    'You love my dog!',\n",
    "    'Do you think my dog is amazing?'\n",
    "]\n",
    "\n",
    "tokenizer = Tokenizer(num_words = 100, oov_token=\"<OOV>\")\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "word_index = tokenizer.word_index\n",
    "\n",
    "sequences = tokenizer.texts_to_sequences(sentences)\n",
    "\n",
    "padded = pad_sequences(sequences, maxlen=5)\n",
    "print(\"\\nWord Index = \" , word_index)\n",
    "print(\"\\nSequences = \" , sequences)\n",
    "print(\"\\nPadded Sequences:\")\n",
    "print(padded)\n",
    "\n",
    "\n",
    "# Try with words that the tokenizer wasn't fit to\n",
    "test_data = [\n",
    "    'i really love my dog',\n",
    "    'my dog loves my manatee'\n",
    "]\n",
    "\n",
    "test_seq = tokenizer.texts_to_sequences(test_data)\n",
    "print(\"\\nTest Sequence = \", test_seq)\n",
    "\n",
    "padded = pad_sequences(test_seq, maxlen=10) #truncated, padding = pre default(change it to post)\n",
    "print(\"\\nPadded Test Sequence: \")\n",
    "print(padded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b301fd94",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}