Format: application/octet-stream
1 {
2 "cells": [
3 {
4 "cell_type": "code",
5 "execution_count": 1,
6 "id": "b274e535",
7 "metadata": {},
8 "outputs": [],
9 "source": [
10 "import tensorflow as tf\n",
11 "from tensorflow import keras\n",
12 "from tensorflow.keras.preprocessing.text import Tokenizer"
13 ]
14 },
15 {
16 "cell_type": "code",
17 "execution_count": 2,
18 "id": "570d8b51",
19 "metadata": {},
20 "outputs": [],
21 "source": [
22 "sentences=['I love everything',\n",
23 " 'i love nothing',\n",
24 " 'Everyone love nothing and everything!',\n",
25 " 'who loves who and why?']"
26 ]
27 },
28 {
29 "cell_type": "code",
30 "execution_count": null,
31 "id": "1da9f851",
32 "metadata": {},
33 "outputs": [],
34 "source": []
35 },
36 {
37 "cell_type": "code",
38 "execution_count": 3,
39 "id": "b5970c09",
40 "metadata": {},
41 "outputs": [],
42 "source": [
43 "toke= Tokenizer(oov_token='<OOV>') #out of vocabulary\n",
44 "toke.fit_on_texts(sentences)\n"
45 ]
46 },
47 {
48 "cell_type": "code",
49 "execution_count": 4,
50 "id": "41640cbe",
51 "metadata": {},
52 "outputs": [],
53 "source": [
54 "word_dict=toke.word_index\n",
55 "seq= toke.texts_to_sequences(sentences)"
56 ]
57 },
58 {
59 "cell_type": "code",
60 "execution_count": null,
61 "id": "15d0d0c4",
62 "metadata": {},
63 "outputs": [],
64 "source": []
65 },
66 {
67 "cell_type": "code",
68 "execution_count": 5,
69 "id": "372f508f",
70 "metadata": {},
71 "outputs": [
72 {
73 "name": "stdout",
74 "output_type": "stream",
75 "text": [
76 "[[3, 2, 4], [3, 2, 5], [8, 2, 5, 6, 4], [7, 9, 7, 6, 10]]\n",
77 "{'<OOV>': 1, 'love': 2, 'i': 3, 'everything': 4, 'nothing': 5, 'and': 6, 'who': 7, 'everyone': 8, 'loves': 9, 'why': 10}\n"
78 ]
79 }
80 ],
81 "source": [
82 "print(seq)\n",
83 "print(word_dict)"
84 ]
85 },
86 {
87 "cell_type": "code",
88 "execution_count": 6,
89 "id": "185e2b90",
90 "metadata": {},
91 "outputs": [],
92 "source": [
93 "from tensorflow.keras.preprocessing.sequence import pad_sequences"
94 ]
95 },
96 {
97 "cell_type": "code",
98 "execution_count": 7,
99 "id": "a9de96d5",
100 "metadata": {},
101 "outputs": [],
102 "source": [
103 "padded= pad_sequences(seq)"
104 ]
105 },
106 {
107 "cell_type": "code",
108 "execution_count": 8,
109 "id": "5dae714f",
110 "metadata": {},
111 "outputs": [
112 {
113 "name": "stdout",
114 "output_type": "stream",
115 "text": [
116 "[[ 0 0 3 2 4]\n",
117 " [ 0 0 3 2 5]\n",
118 " [ 8 2 5 6 4]\n",
119 " [ 7 9 7 6 10]]\n"
120 ]
121 }
122 ],
123 "source": [
124 "print(padded, p)"
125 ]
126 },
127 {
128 "cell_type": "code",
129 "execution_count": null,
130 "id": "4106923b",
131 "metadata": {},
132 "outputs": [],
133 "source": []
134 },
135 {
136 "cell_type": "code",
137 "execution_count": null,
138 "id": "9fe96ad2",
139 "metadata": {},
140 "outputs": [],
141 "source": []
142 },
143 {
144 "cell_type": "code",
145 "execution_count": null,
146 "id": "21842f8e",
147 "metadata": {},
148 "outputs": [],
149 "source": []
150 },
151 {
152 "cell_type": "code",
153 "execution_count": null,
154 "id": "4a60a4c7",
155 "metadata": {},
156 "outputs": [],
157 "source": []
158 },
159 {
160 "cell_type": "code",
161 "execution_count": null,
162 "id": "ba65d624",
163 "metadata": {},
164 "outputs": [],
165 "source": []
166 },
167 {
168 "cell_type": "code",
169 "execution_count": 2,
170 "id": "970d964c",
171 "metadata": {},
172 "outputs": [
173 {
174 "name": "stdout",
175 "output_type": "stream",
176 "text": [
177 "\n",
178 "Word Index = {'<OOV>': 1, 'my': 2, 'love': 3, 'dog': 4, 'i': 5, 'you': 6, 'cat': 7, 'do': 8, 'think': 9, 'is': 10, 'amazing': 11}\n",
179 "\n",
180 "Sequences = [[5, 3, 2, 4], [5, 3, 2, 7], [6, 3, 2, 4], [8, 6, 9, 2, 4, 10, 11]]\n",
181 "\n",
182 "Padded Sequences:\n",
183 "[[ 0 5 3 2 4]\n",
184 " [ 0 5 3 2 7]\n",
185 " [ 0 6 3 2 4]\n",
186 " [ 9 2 4 10 11]]\n",
187 "\n",
188 "Test Sequence = [[5, 1, 3, 2, 4], [2, 4, 1, 2, 1]]\n",
189 "\n",
190 "Padded Test Sequence: \n",
191 "[[0 0 0 0 0 5 1 3 2 4]\n",
192 " [0 0 0 0 0 2 4 1 2 1]]\n"
193 ]
194 }
195 ],
196 "source": [
197 "import tensorflow as tf\n",
198 "from tensorflow import keras\n",
199 "\n",
200 "\n",
201 "from tensorflow.keras.preprocessing.text import Tokenizer\n",
202 "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
203 "\n",
204 "sentences = [\n",
205 " 'I love my dog',\n",
206 " 'I love my cat',\n",
207 " 'You love my dog!',\n",
208 " 'Do you think my dog is amazing?'\n",
209 "]\n",
210 "\n",
211 "tokenizer = Tokenizer(num_words = 100, oov_token=\"<OOV>\")\n",
212 "tokenizer.fit_on_texts(sentences)\n",
213 "word_index = tokenizer.word_index\n",
214 "\n",
215 "sequences = tokenizer.texts_to_sequences(sentences)\n",
216 "\n",
217 "padded = pad_sequences(sequences, maxlen=5)\n",
218 "print(\"\\nWord Index = \" , word_index)\n",
219 "print(\"\\nSequences = \" , sequences)\n",
220 "print(\"\\nPadded Sequences:\")\n",
221 "print(padded)\n",
222 "\n",
223 "\n",
224 "# Try with words that the tokenizer wasn't fit to\n",
225 "test_data = [\n",
226 " 'i really love my dog',\n",
227 " 'my dog loves my manatee'\n",
228 "]\n",
229 "\n",
230 "test_seq = tokenizer.texts_to_sequences(test_data)\n",
231 "print(\"\\nTest Sequence = \", test_seq)\n",
232 "\n",
233 "padded = pad_sequences(test_seq, maxlen=10) #truncated, padding = pre default(change it to post)\n",
234 "print(\"\\nPadded Test Sequence: \")\n",
235 "print(padded)"
236 ]
237 },
238 {
239 "cell_type": "code",
240 "execution_count": null,
241 "id": "b301fd94",
242 "metadata": {},
243 "outputs": [],
244 "source": []
245 }
246 ],
247 "metadata": {
248 "kernelspec": {
249 "display_name": "Python 3",
250 "language": "python",
251 "name": "python3"
252 },
253 "language_info": {
254 "codemirror_mode": {
255 "name": "ipython",
256 "version": 3
257 },
258 "file_extension": ".py",
259 "mimetype": "text/x-python",
260 "name": "python",
261 "nbconvert_exporter": "python",
262 "pygments_lexer": "ipython3",
263 "version": "3.8.8"
264 }
265 },
266 "nbformat": 4,
267 "nbformat_minor": 5
268 }
269