Format: application/octet-stream
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "metadata": {},
6 "source": [
7 "# Module 2 (Python 3)"
8 ]
9 },
10 {
11 "cell_type": "markdown",
12 "metadata": {},
13 "source": [
14 "## Basic NLP Tasks with NLTK"
15 ]
16 },
17 {
18 "cell_type": "code",
19 "execution_count": 1,
20 "metadata": {},
21 "outputs": [
22 {
23 "name": "stdout",
24 "output_type": "stream",
25 "text": [
26 "*** Introductory Examples for the NLTK Book ***\n",
27 "Loading text1, ..., text9 and sent1, ..., sent9\n",
28 "Type the name of the text or sentence to view it.\n",
29 "Type: 'texts()' or 'sents()' to list the materials.\n",
30 "text1: Moby Dick by Herman Melville 1851\n",
31 "text2: Sense and Sensibility by Jane Austen 1811\n",
32 "text3: The Book of Genesis\n",
33 "text4: Inaugural Address Corpus\n",
34 "text5: Chat Corpus\n",
35 "text6: Monty Python and the Holy Grail\n",
36 "text7: Wall Street Journal\n",
37 "text8: Personals Corpus\n",
38 "text9: The Man Who Was Thursday by G . K . Chesterton 1908\n"
39 ]
40 }
41 ],
42 "source": [
43 "import nltk\n",
44 "from nltk.book import *"
45 ]
46 },
47 {
48 "cell_type": "markdown",
49 "metadata": {},
50 "source": [
51 "### Counting vocabulary of words"
52 ]
53 },
54 {
55 "cell_type": "code",
56 "execution_count": 3,
57 "metadata": {},
58 "outputs": [
59 {
60 "data": {
61 "text/plain": [
62 "<Text: Wall Street Journal>"
63 ]
64 },
65 "execution_count": 3,
66 "metadata": {},
67 "output_type": "execute_result"
68 }
69 ],
70 "source": [
71 "text7"
72 ]
73 },
74 {
75 "cell_type": "code",
76 "execution_count": 4,
77 "metadata": {},
78 "outputs": [
79 {
80 "data": {
81 "text/plain": [
82 "['Pierre',\n",
83 " 'Vinken',\n",
84 " ',',\n",
85 " '61',\n",
86 " 'years',\n",
87 " 'old',\n",
88 " ',',\n",
89 " 'will',\n",
90 " 'join',\n",
91 " 'the',\n",
92 " 'board',\n",
93 " 'as',\n",
94 " 'a',\n",
95 " 'nonexecutive',\n",
96 " 'director',\n",
97 " 'Nov.',\n",
98 " '29',\n",
99 " '.']"
100 ]
101 },
102 "execution_count": 4,
103 "metadata": {},
104 "output_type": "execute_result"
105 }
106 ],
107 "source": [
108 "sent7"
109 ]
110 },
111 {
112 "cell_type": "code",
113 "execution_count": 5,
114 "metadata": {},
115 "outputs": [
116 {
117 "data": {
118 "text/plain": [
119 "18"
120 ]
121 },
122 "execution_count": 5,
123 "metadata": {},
124 "output_type": "execute_result"
125 }
126 ],
127 "source": [
128 "len(sent7)"
129 ]
130 },
131 {
132 "cell_type": "code",
133 "execution_count": 6,
134 "metadata": {},
135 "outputs": [
136 {
137 "data": {
138 "text/plain": [
139 "100676"
140 ]
141 },
142 "execution_count": 6,
143 "metadata": {},
144 "output_type": "execute_result"
145 }
146 ],
147 "source": [
148 "len(text7)"
149 ]
150 },
151 {
152 "cell_type": "code",
153 "execution_count": 7,
154 "metadata": {},
155 "outputs": [
156 {
157 "data": {
158 "text/plain": [
159 "12408"
160 ]
161 },
162 "execution_count": 7,
163 "metadata": {},
164 "output_type": "execute_result"
165 }
166 ],
167 "source": [
168 "len(set(text7))"
169 ]
170 },
171 {
172 "cell_type": "code",
173 "execution_count": 8,
174 "metadata": {},
175 "outputs": [
176 {
177 "data": {
178 "text/plain": [
179 "['bottom',\n",
180 " 'Richmond',\n",
181 " 'tension',\n",
182 " 'limits',\n",
183 " 'Wedtech',\n",
184 " 'most',\n",
185 " 'boost',\n",
186 " '143.80',\n",
187 " 'Dale',\n",
188 " 'refunded']"
189 ]
190 },
191 "execution_count": 8,
192 "metadata": {},
193 "output_type": "execute_result"
194 }
195 ],
196 "source": [
197 "list(set(text7))[:10]"
198 ]
199 },
200 {
201 "cell_type": "markdown",
202 "metadata": {},
203 "source": [
204 "### Frequency of words"
205 ]
206 },
207 {
208 "cell_type": "code",
209 "execution_count": 9,
210 "metadata": {},
211 "outputs": [
212 {
213 "data": {
214 "text/plain": [
215 "12408"
216 ]
217 },
218 "execution_count": 9,
219 "metadata": {},
220 "output_type": "execute_result"
221 }
222 ],
223 "source": [
224 "dist = FreqDist(text7)\n",
225 "len(dist)"
226 ]
227 },
228 {
229 "cell_type": "code",
230 "execution_count": 15,
231 "metadata": {},
232 "outputs": [
233 {
234 "data": {
235 "text/plain": [
236 "['Pierre', 'Vinken', ',', '61', 'years', 'old', 'will', 'join', 'the', 'board']"
237 ]
238 },
239 "execution_count": 15,
240 "metadata": {},
241 "output_type": "execute_result"
242 }
243 ],
244 "source": [
245 "vocab1 = dist.keys()\n",
246 "#vocab1[:10] \n",
247 "# In Python 3 dict.keys() returns an iterable view instead of a list\n",
248 "list(vocab1)[:10]"
249 ]
250 },
251 {
252 "cell_type": "code",
253 "execution_count": 16,
254 "metadata": {},
255 "outputs": [
256 {
257 "data": {
258 "text/plain": [
259 "20"
260 ]
261 },
262 "execution_count": 16,
263 "metadata": {},
264 "output_type": "execute_result"
265 }
266 ],
267 "source": [
268 "dist['four']"
269 ]
270 },
271 {
272 "cell_type": "code",
273 "execution_count": 17,
274 "metadata": {},
275 "outputs": [
276 {
277 "data": {
278 "text/plain": [
279 "['billion',\n",
280 " 'company',\n",
281 " 'president',\n",
282 " 'because',\n",
283 " 'market',\n",
284 " 'million',\n",
285 " 'shares',\n",
286 " 'trading',\n",
287 " 'program']"
288 ]
289 },
290 "execution_count": 17,
291 "metadata": {},
292 "output_type": "execute_result"
293 }
294 ],
295 "source": [
296 "freqwords = [w for w in vocab1 if len(w) > 5 and dist[w] > 100]\n",
297 "freqwords"
298 ]
299 },
300 {
301 "cell_type": "markdown",
302 "metadata": {},
303 "source": [
304 "### Normalization and stemming"
305 ]
306 },
307 {
308 "cell_type": "code",
309 "execution_count": 22,
310 "metadata": {},
311 "outputs": [
312 {
313 "data": {
314 "text/plain": [
315 "['list', 'listed', 'lists', 'listing', 'listings']"
316 ]
317 },
318 "execution_count": 22,
319 "metadata": {},
320 "output_type": "execute_result"
321 }
322 ],
323 "source": [
324 "input1 = \"List listed lists listing listings\"\n",
325 "words1 = input1.lower().split(' ')\n",
326 "words1"
327 ]
328 },
329 {
330 "cell_type": "code",
331 "execution_count": 23,
332 "metadata": {},
333 "outputs": [
334 {
335 "data": {
336 "text/plain": [
337 "['list', 'list', 'list', 'list', 'list']"
338 ]
339 },
340 "execution_count": 23,
341 "metadata": {},
342 "output_type": "execute_result"
343 }
344 ],
345 "source": [
346 "porter = nltk.PorterStemmer()\n",
347 "[porter.stem(t) for t in words1]"
348 ]
349 },
350 {
351 "cell_type": "markdown",
352 "metadata": {},
353 "source": [
354 "### Lemmatization"
355 ]
356 },
357 {
358 "cell_type": "code",
359 "execution_count": 26,
360 "metadata": {},
361 "outputs": [
362 {
363 "data": {
364 "text/plain": [
365 "['Universal',\n",
366 " 'Declaration',\n",
367 " 'of',\n",
368 " 'Human',\n",
369 " 'Rights',\n",
370 " 'Preamble',\n",
371 " 'Whereas',\n",
372 " 'recognition',\n",
373 " 'of',\n",
374 " 'the',\n",
375 " 'inherent',\n",
376 " 'dignity',\n",
377 " 'and',\n",
378 " 'of',\n",
379 " 'the',\n",
380 " 'equal',\n",
381 " 'and',\n",
382 " 'inalienable',\n",
383 " 'rights',\n",
384 " 'of']"
385 ]
386 },
387 "execution_count": 26,
388 "metadata": {},
389 "output_type": "execute_result"
390 }
391 ],
392 "source": [
393 "udhr = nltk.corpus.udhr.words('English-Latin1')\n",
394 "udhr[:20]"
395 ]
396 },
397 {
398 "cell_type": "code",
399 "execution_count": 24,
400 "metadata": {},
401 "outputs": [
402 {
403 "data": {
404 "text/plain": [
405 "['univers',\n",
406 " 'declar',\n",
407 " 'of',\n",
408 " 'human',\n",
409 " 'right',\n",
410 " 'preambl',\n",
411 " 'wherea',\n",
412 " 'recognit',\n",
413 " 'of',\n",
414 " 'the',\n",
415 " 'inher',\n",
416 " 'digniti',\n",
417 " 'and',\n",
418 " 'of',\n",
419 " 'the',\n",
420 " 'equal',\n",
421 " 'and',\n",
422 " 'inalien',\n",
423 " 'right',\n",
424 " 'of']"
425 ]
426 },
427 "execution_count": 24,
428 "metadata": {},
429 "output_type": "execute_result"
430 }
431 ],
432 "source": [
433 "[porter.stem(t) for t in udhr[:20]] # Still Lemmatization"
434 ]
435 },
436 {
437 "cell_type": "code",
438 "execution_count": 25,
439 "metadata": {},
440 "outputs": [
441 {
442 "data": {
443 "text/plain": [
444 "['Universal',\n",
445 " 'Declaration',\n",
446 " 'of',\n",
447 " 'Human',\n",
448 " 'Rights',\n",
449 " 'Preamble',\n",
450 " 'Whereas',\n",
451 " 'recognition',\n",
452 " 'of',\n",
453 " 'the',\n",
454 " 'inherent',\n",
455 " 'dignity',\n",
456 " 'and',\n",
457 " 'of',\n",
458 " 'the',\n",
459 " 'equal',\n",
460 " 'and',\n",
461 " 'inalienable',\n",
462 " 'right',\n",
463 " 'of']"
464 ]
465 },
466 "execution_count": 25,
467 "metadata": {},
468 "output_type": "execute_result"
469 }
470 ],
471 "source": [
472 "WNlemma = nltk.WordNetLemmatizer()\n",
473 "[WNlemma.lemmatize(t) for t in udhr[:20]]"
474 ]
475 },
476 {
477 "cell_type": "markdown",
478 "metadata": {},
479 "source": [
480 "### Tokenization"
481 ]
482 },
483 {
484 "cell_type": "code",
485 "execution_count": 28,
486 "metadata": {},
487 "outputs": [
488 {
489 "data": {
490 "text/plain": [
491 "['Children', \"shouldn't\", 'drink', 'a', 'sugary', 'drink', 'before', 'bed.']"
492 ]
493 },
494 "execution_count": 28,
495 "metadata": {},
496 "output_type": "execute_result"
497 }
498 ],
499 "source": [
500 "text11 = \"Children shouldn't drink a sugary drink before bed.\"\n",
501 "text11.split(' ')"
502 ]
503 },
504 {
505 "cell_type": "code",
506 "execution_count": 29,
507 "metadata": {},
508 "outputs": [
509 {
510 "data": {
511 "text/plain": [
512 "['Children',\n",
513 " 'should',\n",
514 " \"n't\",\n",
515 " 'drink',\n",
516 " 'a',\n",
517 " 'sugary',\n",
518 " 'drink',\n",
519 " 'before',\n",
520 " 'bed',\n",
521 " '.']"
522 ]
523 },
524 "execution_count": 29,
525 "metadata": {},
526 "output_type": "execute_result"
527 }
528 ],
529 "source": [
530 "nltk.word_tokenize(text11)"
531 ]
532 },
533 {
534 "cell_type": "code",
535 "execution_count": 30,
536 "metadata": {},
537 "outputs": [
538 {
539 "data": {
540 "text/plain": [
541 "4"
542 ]
543 },
544 "execution_count": 30,
545 "metadata": {},
546 "output_type": "execute_result"
547 }
548 ],
549 "source": [
550 "text12 = \"This is the first sentence. A gallon of milk in the U.S. costs $2.99. Is this the third sentence? Yes, it is!\"\n",
551 "sentences = nltk.sent_tokenize(text12)\n",
552 "len(sentences)"
553 ]
554 },
555 {
556 "cell_type": "code",
557 "execution_count": 31,
558 "metadata": {},
559 "outputs": [
560 {
561 "data": {
562 "text/plain": [
563 "['This is the first sentence.',\n",
564 " 'A gallon of milk in the U.S. costs $2.99.',\n",
565 " 'Is this the third sentence?',\n",
566 " 'Yes, it is!']"
567 ]
568 },
569 "execution_count": 31,
570 "metadata": {},
571 "output_type": "execute_result"
572 }
573 ],
574 "source": [
575 "sentences"
576 ]
577 },
578 {
579 "cell_type": "markdown",
580 "metadata": {},
581 "source": [
582 "## Advanced NLP Tasks with NLTK"
583 ]
584 },
585 {
586 "cell_type": "markdown",
587 "metadata": {},
588 "source": [
589 "### POS tagging"
590 ]
591 },
592 {
593 "cell_type": "code",
594 "execution_count": 33,
595 "metadata": {},
596 "outputs": [
597 {
598 "name": "stdout",
599 "output_type": "stream",
600 "text": [
601 "MD: modal auxiliary\n",
602 " can cannot could couldn't dare may might must need ought shall should\n",
603 " shouldn't will would\n"
604 ]
605 }
606 ],
607 "source": [
608 "nltk.help.upenn_tagset('MD')"
609 ]
610 },
611 {
612 "cell_type": "code",
613 "execution_count": 34,
614 "metadata": {},
615 "outputs": [
616 {
617 "data": {
618 "text/plain": [
619 "[('Children', 'NNP'),\n",
620 " ('should', 'MD'),\n",
621 " (\"n't\", 'RB'),\n",
622 " ('drink', 'VB'),\n",
623 " ('a', 'DT'),\n",
624 " ('sugary', 'JJ'),\n",
625 " ('drink', 'NN'),\n",
626 " ('before', 'IN'),\n",
627 " ('bed', 'NN'),\n",
628 " ('.', '.')]"
629 ]
630 },
631 "execution_count": 34,
632 "metadata": {},
633 "output_type": "execute_result"
634 }
635 ],
636 "source": [
637 "text13 = nltk.word_tokenize(text11)\n",
638 "nltk.pos_tag(text13)"
639 ]
640 },
641 {
642 "cell_type": "code",
643 "execution_count": 35,
644 "metadata": {},
645 "outputs": [
646 {
647 "data": {
648 "text/plain": [
649 "[('Visiting', 'VBG'),\n",
650 " ('aunts', 'NNS'),\n",
651 " ('can', 'MD'),\n",
652 " ('be', 'VB'),\n",
653 " ('a', 'DT'),\n",
654 " ('nuisance', 'NN')]"
655 ]
656 },
657 "execution_count": 35,
658 "metadata": {},
659 "output_type": "execute_result"
660 }
661 ],
662 "source": [
663 "text14 = nltk.word_tokenize(\"Visiting aunts can be a nuisance\")\n",
664 "nltk.pos_tag(text14)"
665 ]
666 },
667 {
668 "cell_type": "code",
669 "execution_count": 37,
670 "metadata": {},
671 "outputs": [
672 {
673 "name": "stdout",
674 "output_type": "stream",
675 "text": [
676 "(S (NP Alice) (VP (V loves) (NP Bob)))\n"
677 ]
678 }
679 ],
680 "source": [
681 "# Parsing sentence structure\n",
682 "text15 = nltk.word_tokenize(\"Alice loves Bob\")\n",
683 "grammar = nltk.CFG.fromstring(\"\"\"\n",
684 "S -> NP VP\n",
685 "VP -> V NP\n",
686 "NP -> 'Alice' | 'Bob'\n",
687 "V -> 'loves'\n",
688 "\"\"\")\n",
689 "\n",
690 "parser = nltk.ChartParser(grammar)\n",
691 "trees = parser.parse_all(text15)\n",
692 "for tree in trees:\n",
693 " print(tree)"
694 ]
695 },
696 {
697 "cell_type": "code",
698 "execution_count": 40,
699 "metadata": {},
700 "outputs": [
701 {
702 "data": {
703 "text/plain": [
704 "<Grammar with 13 productions>"
705 ]
706 },
707 "execution_count": 40,
708 "metadata": {},
709 "output_type": "execute_result"
710 }
711 ],
712 "source": [
713 "text16 = nltk.word_tokenize(\"I saw the man with a telescope\")\n",
714 "grammar1 = nltk.data.load('mygrammar.cfg')\n",
715 "grammar1"
716 ]
717 },
718 {
719 "cell_type": "code",
720 "execution_count": 41,
721 "metadata": {},
722 "outputs": [
723 {
724 "name": "stdout",
725 "output_type": "stream",
726 "text": [
727 "(S\n",
728 " (NP I)\n",
729 " (VP\n",
730 " (VP (V saw) (NP (Det the) (N man)))\n",
731 " (PP (P with) (NP (Det a) (N telescope)))))\n",
732 "(S\n",
733 " (NP I)\n",
734 " (VP\n",
735 " (V saw)\n",
736 " (NP (Det the) (N man) (PP (P with) (NP (Det a) (N telescope))))))\n"
737 ]
738 }
739 ],
740 "source": [
741 "parser = nltk.ChartParser(grammar1)\n",
742 "trees = parser.parse_all(text16)\n",
743 "for tree in trees:\n",
744 " print(tree)"
745 ]
746 },
747 {
748 "cell_type": "code",
749 "execution_count": 42,
750 "metadata": {},
751 "outputs": [
752 {
753 "name": "stdout",
754 "output_type": "stream",
755 "text": [
756 "(S\n",
757 " (NP-SBJ\n",
758 " (NP (NNP Pierre) (NNP Vinken))\n",
759 " (, ,)\n",
760 " (ADJP (NP (CD 61) (NNS years)) (JJ old))\n",
761 " (, ,))\n",
762 " (VP\n",
763 " (MD will)\n",
764 " (VP\n",
765 " (VB join)\n",
766 " (NP (DT the) (NN board))\n",
767 " (PP-CLR (IN as) (NP (DT a) (JJ nonexecutive) (NN director)))\n",
768 " (NP-TMP (NNP Nov.) (CD 29))))\n",
769 " (. .))\n"
770 ]
771 }
772 ],
773 "source": [
774 "from nltk.corpus import treebank\n",
775 "text17 = treebank.parsed_sents('wsj_0001.mrg')[0]\n",
776 "print(text17)"
777 ]
778 },
779 {
780 "cell_type": "markdown",
781 "metadata": {},
782 "source": [
783 "### POS tagging and parsing ambiguity"
784 ]
785 },
786 {
787 "cell_type": "code",
788 "execution_count": 43,
789 "metadata": {},
790 "outputs": [
791 {
792 "data": {
793 "text/plain": [
794 "[('The', 'DT'), ('old', 'JJ'), ('man', 'NN'), ('the', 'DT'), ('boat', 'NN')]"
795 ]
796 },
797 "execution_count": 43,
798 "metadata": {},
799 "output_type": "execute_result"
800 }
801 ],
802 "source": [
803 "text18 = nltk.word_tokenize(\"The old man the boat\")\n",
804 "nltk.pos_tag(text18)"
805 ]
806 },
807 {
808 "cell_type": "code",
809 "execution_count": 44,
810 "metadata": {},
811 "outputs": [
812 {
813 "data": {
814 "text/plain": [
815 "[('Colorless', 'NNP'),\n",
816 " ('green', 'JJ'),\n",
817 " ('ideas', 'NNS'),\n",
818 " ('sleep', 'VBP'),\n",
819 " ('furiously', 'RB')]"
820 ]
821 },
822 "execution_count": 44,
823 "metadata": {},
824 "output_type": "execute_result"
825 }
826 ],
827 "source": [
828 "text19 = nltk.word_tokenize(\"Colorless green ideas sleep furiously\")\n",
829 "nltk.pos_tag(text19)"
830 ]
831 }
832 ],
833 "metadata": {
834 "kernelspec": {
835 "display_name": "Python 3",
836 "language": "python",
837 "name": "python3"
838 },
839 "language_info": {
840 "codemirror_mode": {
841 "name": "ipython",
842 "version": 3
843 },
844 "file_extension": ".py",
845 "mimetype": "text/x-python",
846 "name": "python",
847 "nbconvert_exporter": "python",
848 "pygments_lexer": "ipython3",
849 "version": "3.6.2"
850 }
851 },
852 "nbformat": 4,
853 "nbformat_minor": 2
854 }
855