Format: application/octet-stream
1 {
2 "cells": [
3 {
4 "cell_type": "code",
5 "execution_count": 1,
6 "id": "b9900828",
7 "metadata": {},
8 "outputs": [
9 {
10 "name": "stderr",
11 "output_type": "stream",
12 "text": [
13 "C:\\Users\\PC\\anaconda3\\lib\\site-packages\\gensim\\similarities\\__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package <https://pypi.org/project/python-Levenshtein/> is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n",
14 " warnings.warn(msg)\n"
15 ]
16 }
17 ],
18 "source": [
19 "import pandas as pd\n",
20 "import spacy\n",
21 "import gensim.corpora as co\n",
22 "from gensim.utils import simple_preprocess\n",
23 "from gensim.models import CoherenceModel\n",
24 "import pyLDAvis\n",
25 "import pyLDAvis.gensim_models\n",
26 "from nltk.corpus import stopwords\n",
27 "import gensim\n",
28 "import warnings\n",
29 "warnings.filterwarnings(\"ignore\", category=DeprecationWarning) "
30 ]
31 },
32 {
33 "cell_type": "code",
34 "execution_count": 2,
35 "id": "386fb83e",
36 "metadata": {},
37 "outputs": [],
38 "source": [
39 "plots=pd.read_csv('tagsText.tsv', sep='\\t')\n",
40 "plots1=pd.read_csv('tagsText1.tsv', sep='\\t')"
41 ]
42 },
43 {
44 "cell_type": "code",
45 "execution_count": 3,
46 "id": "6628dac9",
47 "metadata": {},
48 "outputs": [],
49 "source": [
50 "plots=plots.append(plots1, ignore_index=True)"
51 ]
52 },
53 {
54 "cell_type": "code",
55 "execution_count": 5,
56 "id": "878f2dbb",
57 "metadata": {},
58 "outputs": [],
59 "source": [
60 "#plots.drop('Unnamed: 0', axis=1, inplace=True)"
61 ]
62 },
63 {
64 "cell_type": "code",
65 "execution_count": 6,
66 "id": "9dfc4f80",
67 "metadata": {},
68 "outputs": [
69 {
70 "data": {
71 "text/html": [
72 "<div>\n",
73 "<style scoped>\n",
74 " .dataframe tbody tr th:only-of-type {\n",
75 " vertical-align: middle;\n",
76 " }\n",
77 "\n",
78 " .dataframe tbody tr th {\n",
79 " vertical-align: top;\n",
80 " }\n",
81 "\n",
82 " .dataframe thead th {\n",
83 " text-align: right;\n",
84 " }\n",
85 "</style>\n",
86 "<table border=\"1\" class=\"dataframe\">\n",
87 " <thead>\n",
88 " <tr style=\"text-align: right;\">\n",
89 " <th></th>\n",
90 " <th>0</th>\n",
91 " </tr>\n",
92 " </thead>\n",
93 " <tbody>\n",
94 " <tr>\n",
95 " <th>0</th>\n",
96 " <td>wrongful imprisonment based on the works of st...</td>\n",
97 " </tr>\n",
98 " <tr>\n",
99 " <th>1</th>\n",
100 " <td>dc comics joker psychopath clown criminal mast...</td>\n",
101 " </tr>\n",
102 " <tr>\n",
103 " <th>2</th>\n",
104 " <td>dream ambiguous ending subconscious surprise e...</td>\n",
105 " </tr>\n",
106 " <tr>\n",
107 " <th>3</th>\n",
108 " <td>fighting surprise ending anti establishment in...</td>\n",
109 " </tr>\n",
110 " <tr>\n",
111 " <th>4</th>\n",
112 " <td>nonlinear timeline overdose drug overdose drug...</td>\n",
113 " </tr>\n",
114 " <tr>\n",
115 " <th>...</th>\n",
116 " <td>...</td>\n",
117 " </tr>\n",
118 " <tr>\n",
119 " <th>3995</th>\n",
120 " <td>surrealism donkey breast fondling female nudit...</td>\n",
121 " </tr>\n",
122 " <tr>\n",
123 " <th>3996</th>\n",
124 " <td>golf nickname love kiss friend rivalry texas p...</td>\n",
125 " </tr>\n",
126 " <tr>\n",
127 " <th>3997</th>\n",
128 " <td>baseball year 1932 wrongful conviction passage...</td>\n",
129 " </tr>\n",
130 " <tr>\n",
131 " <th>3998</th>\n",
132 " <td>generation x houston texas texas nipples visib...</td>\n",
133 " </tr>\n",
134 " <tr>\n",
135 " <th>3999</th>\n",
136 " <td>f rated revisionist history triple f rated fem...</td>\n",
137 " </tr>\n",
138 " </tbody>\n",
139 "</table>\n",
140 "<p>4000 rows × 1 columns</p>\n",
141 "</div>"
142 ],
143 "text/plain": [
144 " 0\n",
145 "0 wrongful imprisonment based on the works of st...\n",
146 "1 dc comics joker psychopath clown criminal mast...\n",
147 "2 dream ambiguous ending subconscious surprise e...\n",
148 "3 fighting surprise ending anti establishment in...\n",
149 "4 nonlinear timeline overdose drug overdose drug...\n",
150 "... ...\n",
151 "3995 surrealism donkey breast fondling female nudit...\n",
152 "3996 golf nickname love kiss friend rivalry texas p...\n",
153 "3997 baseball year 1932 wrongful conviction passage...\n",
154 "3998 generation x houston texas texas nipples visib...\n",
155 "3999 f rated revisionist history triple f rated fem...\n",
156 "\n",
157 "[4000 rows x 1 columns]"
158 ]
159 },
160 "execution_count": 6,
161 "metadata": {},
162 "output_type": "execute_result"
163 }
164 ],
165 "source": [
166 "plots"
167 ]
168 },
169 {
170 "cell_type": "code",
171 "execution_count": null,
172 "id": "a0706973",
173 "metadata": {},
174 "outputs": [],
175 "source": []
176 },
177 {
178 "cell_type": "code",
179 "execution_count": 7,
180 "id": "22f15ce3",
181 "metadata": {},
182 "outputs": [],
183 "source": [
184 "texts= plots.to_numpy()"
185 ]
186 },
187 {
188 "cell_type": "code",
189 "execution_count": 8,
190 "id": "ac964ab2",
191 "metadata": {},
192 "outputs": [],
193 "source": [
194 "texts=texts.reshape(-1)"
195 ]
196 },
197 {
198 "cell_type": "code",
199 "execution_count": 9,
200 "id": "86341e9d",
201 "metadata": {},
202 "outputs": [],
203 "source": [
204 "#plots['texts']= plots['0'].astype(str)+' '+plots['1'].astype(str)+' '+plots['2'].astype(str)+' '+plots['3'].astype(str)+' '+plots['4'].astype(str)+' '+plots['5'].astype(str)\\\n",
205 " # +' '+plots['6'].astype(str)+' '+plots['7'].astype(str)+' '+plots['8'].astype(str)"
206 ]
207 },
208 {
209 "cell_type": "code",
210 "execution_count": null,
211 "id": "f490d8a6",
212 "metadata": {},
213 "outputs": [],
214 "source": [
215 "texts[0]"
216 ]
217 },
218 {
219 "cell_type": "code",
220 "execution_count": null,
221 "id": "d3c5cfda",
222 "metadata": {},
223 "outputs": [],
224 "source": []
225 },
226 {
227 "cell_type": "code",
228 "execution_count": null,
229 "id": "578a16b2",
230 "metadata": {},
231 "outputs": [],
232 "source": []
233 },
234 {
235 "cell_type": "code",
236 "execution_count": null,
237 "id": "81e9ffcd",
238 "metadata": {},
239 "outputs": [],
240 "source": []
241 },
242 {
243 "cell_type": "code",
244 "execution_count": null,
245 "id": "563668e6",
246 "metadata": {},
247 "outputs": [],
248 "source": []
249 },
250 {
251 "cell_type": "code",
252 "execution_count": null,
253 "id": "bd825ecb",
254 "metadata": {},
255 "outputs": [],
256 "source": []
257 },
258 {
259 "cell_type": "code",
260 "execution_count": null,
261 "id": "74a3288f",
262 "metadata": {},
263 "outputs": [],
264 "source": [
265 "stopwords= stopwords.words('english')"
266 ]
267 },
268 {
269 "cell_type": "code",
270 "execution_count": null,
271 "id": "4d02b97c",
272 "metadata": {},
273 "outputs": [],
274 "source": [
275 "stopwords.extend(['go','none'])"
276 ]
277 },
278 {
279 "cell_type": "code",
280 "execution_count": null,
281 "id": "02689294",
282 "metadata": {},
283 "outputs": [],
284 "source": []
285 },
286 {
287 "cell_type": "code",
288 "execution_count": null,
289 "id": "7f1d8ccf",
290 "metadata": {},
291 "outputs": [],
292 "source": []
293 },
294 {
295 "cell_type": "code",
296 "execution_count": null,
297 "id": "818eea61",
298 "metadata": {},
299 "outputs": [],
300 "source": []
301 },
302 {
303 "cell_type": "code",
304 "execution_count": null,
305 "id": "eb58b4e7",
306 "metadata": {},
307 "outputs": [],
308 "source": [
309 "def lemma(texts, allowed_pos=['NOUN',\"ADJ\",'JJ','JJR', 'VB']):\n",
310 " nlp= spacy.load('en_core_web_sm', disable=['parser', 'ner'])\n",
311 " text_out=[]\n",
312 " i=1\n",
313 " for text in texts:\n",
314 " print(i)\n",
315 " doc= nlp(text)\n",
316 " #print(doc)\n",
317 " new_text=[]\n",
318 " for token in doc:\n",
319 " if token.text not in stopwords:\n",
320 " #if token.pos_ in allowed_pos:\n",
321 " #print('\\nt***\\n',token.lemma_)\n",
322 " new_text.append(token.lemma_)\n",
323 " \n",
324 " final=' '.join(new_text)\n",
325 " #print('\\n***\\n',final)\n",
326 " text_out.append(final)\n",
327 " i+=1\n",
328 " \n",
329 " return text_out\n",
330 " \n",
331 "lemma_texts= lemma(texts) "
332 ]
333 },
334 {
335 "cell_type": "code",
336 "execution_count": null,
337 "id": "f6c738fe",
338 "metadata": {},
339 "outputs": [],
340 "source": [
341 "lemma_texts=[]\n",
342 "for text in texts:\n",
343 " lemma_texts.append(''.join(str(text)))"
344 ]
345 },
346 {
347 "cell_type": "markdown",
348 "id": "d6eaf1ab",
349 "metadata": {},
350 "source": [
351 "import joblib\n",
352 "\n",
353 "lemma_texts=joblib.load('lemma_texts_plots.pkl')"
354 ]
355 },
356 {
357 "cell_type": "code",
358 "execution_count": null,
359 "id": "5dd67c7d",
360 "metadata": {},
361 "outputs": [],
362 "source": [
363 "lemma_texts[0]"
364 ]
365 },
366 {
367 "cell_type": "code",
368 "execution_count": null,
369 "id": "76838401",
370 "metadata": {},
371 "outputs": [],
372 "source": [
373 "texts[0]"
374 ]
375 },
376 {
377 "cell_type": "code",
378 "execution_count": null,
379 "id": "f2ca57c2",
380 "metadata": {},
381 "outputs": [],
382 "source": []
383 },
384 {
385 "cell_type": "code",
386 "execution_count": null,
387 "id": "223ea0e2",
388 "metadata": {},
389 "outputs": [],
390 "source": []
391 },
392 {
393 "cell_type": "code",
394 "execution_count": null,
395 "id": "ba169168",
396 "metadata": {},
397 "outputs": [],
398 "source": []
399 },
400 {
401 "cell_type": "code",
402 "execution_count": null,
403 "id": "ada922e4",
404 "metadata": {},
405 "outputs": [],
406 "source": [
407 "def gen_words(texts):\n",
408 " final=[]\n",
409 " i=1\n",
410 " for text in texts:\n",
411 " print(i)\n",
412 " new= simple_preprocess(text, deacc= True)\n",
413 " final.append(new)\n",
414 " i+=1\n",
415 " return final\n",
416 "data_words= gen_words(lemma_texts)"
417 ]
418 },
419 {
420 "cell_type": "code",
421 "execution_count": null,
422 "id": "86be7967",
423 "metadata": {},
424 "outputs": [],
425 "source": []
426 },
427 {
428 "cell_type": "code",
429 "execution_count": null,
430 "id": "19ef81be",
431 "metadata": {},
432 "outputs": [],
433 "source": []
434 },
435 {
436 "cell_type": "code",
437 "execution_count": null,
438 "id": "8a03c4c2",
439 "metadata": {},
440 "outputs": [],
441 "source": [
442 "# Build the bigram and trigram models\n",
443 "bigram = gensim.models.Phrases(data_words, min_count=3, threshold=50) # higher threshold fewer phrases.\n",
444 "trigram = gensim.models.Phrases(bigram[data_words], threshold=50) \n",
445 "\n",
446 "# Faster way to get a sentence clubbed as a trigram/bigram\n",
447 "bigram = gensim.models.phrases.Phraser(bigram)\n",
448 "trigram = gensim.models.phrases.Phraser(trigram)\n",
449 "\n",
450 "def make_bi(texts):\n",
451 " return ([bigram[doc] for doc in texts])\n",
452 "\n",
453 "def make_tri(texts):\n",
454 " return ([trigram[doc] for doc in texts])\n",
455 "\n",
456 "data_bigrams= make_bi(data_words)\n",
457 "data_bi_tri= make_tri(data_bigrams)"
458 ]
459 },
460 {
461 "cell_type": "code",
462 "execution_count": null,
463 "id": "e80ff155",
464 "metadata": {},
465 "outputs": [],
466 "source": []
467 },
468 {
469 "cell_type": "code",
470 "execution_count": null,
471 "id": "bdc45c2c",
472 "metadata": {},
473 "outputs": [],
474 "source": [
475 "print(texts[0])"
476 ]
477 },
478 {
479 "cell_type": "code",
480 "execution_count": null,
481 "id": "f4bb3979",
482 "metadata": {},
483 "outputs": [],
484 "source": []
485 },
486 {
487 "cell_type": "code",
488 "execution_count": null,
489 "id": "e3b28b9b",
490 "metadata": {},
491 "outputs": [],
492 "source": []
493 },
494 {
495 "cell_type": "code",
496 "execution_count": null,
497 "id": "1614ce76",
498 "metadata": {},
499 "outputs": [],
500 "source": []
501 },
502 {
503 "cell_type": "code",
504 "execution_count": null,
505 "id": "2cbeeda7",
506 "metadata": {},
507 "outputs": [],
508 "source": [
509 "from gensim.models import TfidfModel\n",
510 "\n",
511 "id2word= co.Dictionary(texts.astype(str))\n",
512 "\n",
513 "\n",
514 "\n",
515 "corpus= [ id2word.doc2bow(str(text)) for text in texts]\n",
516 "\n",
517 "tfidf = TfidfModel(corpus, id2word)\n",
518 "\n",
519 "low_value= 0.02\n",
520 "words=[]\n",
521 "words_missin_in_tfidf=[]\n",
522 "\n",
523 "\n",
524 "for i in range(0, len(corpus)):\n",
525 " bow = corpus[i]\n",
526 " #print(bow[:30])\n",
527 " low_value_words = [] #reinitialize to be safe. You can skip this.\n",
528 " low_value_words = [id for id, value in tfidf[bow] if value < low_value]\n",
529 " new_bow = [b for b in bow if b[0] not in low_value_words]\n",
530 "\n",
531 " #reassign \n",
532 " corpus[i] = new_bow"
533 ]
534 },
535 {
536 "cell_type": "code",
537 "execution_count": null,
538 "id": "d75d3d74",
539 "metadata": {},
540 "outputs": [],
541 "source": []
542 },
543 {
544 "cell_type": "markdown",
545 "id": "98d281b0",
546 "metadata": {},
547 "source": [
548 "id2word= co.Dictionary(data_words)\n",
549 "def corpus_d(data_words, id2word):\n",
550 " corpus=[]\n",
551 " i=1\n",
552 " for text in data_words:\n",
553 " print(i)\n",
554 " corpus.append(id2word.doc2bow(text))\n",
555 " i+=1\n",
556 " return corpus\n",
557 "\n",
558 "\n",
559 "corpus= corpus_d(data_words, id2word)"
560 ]
561 },
562 {
563 "cell_type": "code",
564 "execution_count": null,
565 "id": "0f261167",
566 "metadata": {},
567 "outputs": [],
568 "source": [
569 "len(corpus)"
570 ]
571 },
572 {
573 "cell_type": "code",
574 "execution_count": null,
575 "id": "ae7c1507",
576 "metadata": {},
577 "outputs": [],
578 "source": []
579 },
580 {
581 "cell_type": "code",
582 "execution_count": null,
583 "id": "f9a9f8b2",
584 "metadata": {},
585 "outputs": [],
586 "source": []
587 },
588 {
589 "cell_type": "code",
590 "execution_count": null,
591 "id": "47211ee0",
592 "metadata": {},
593 "outputs": [],
594 "source": [
595 "lda= gensim.models.ldamodel.LdaModel(corpus=corpus,\n",
596 " num_topics=11,\n",
597 " id2word=id2word,\n",
598 " chunksize=200,\n",
599 " passes=10,\n",
600 " update_every=1,\n",
601 " alpha='auto',\n",
602 " random_state=100)"
603 ]
604 },
605 {
606 "cell_type": "code",
607 "execution_count": null,
608 "id": "5eda0c14",
609 "metadata": {},
610 "outputs": [],
611 "source": []
612 },
613 {
614 "cell_type": "code",
615 "execution_count": null,
616 "id": "8a9aafca",
617 "metadata": {},
618 "outputs": [],
619 "source": []
620 },
621 {
622 "cell_type": "code",
623 "execution_count": null,
624 "id": "e4203c3a",
625 "metadata": {},
626 "outputs": [],
627 "source": [
628 "pyLDAvis.enable_notebook()\n",
629 "vis= pyLDAvis.gensim_models.prepare(lda, corpus, id2word, mds='mmds', R=30)\n",
630 "vis"
631 ]
632 },
633 {
634 "cell_type": "code",
635 "execution_count": null,
636 "id": "75411fa6",
637 "metadata": {
638 "scrolled": true
639 },
640 "outputs": [],
641 "source": []
642 },
643 {
644 "cell_type": "code",
645 "execution_count": null,
646 "id": "a2c4d880",
647 "metadata": {},
648 "outputs": [],
649 "source": [
650 "from gensim.models import CoherenceModel\n",
651 "# Compute Coherence Score\n",
652 "coherence_model_lda = CoherenceModel(model=lda, texts=data_bi_tri, dictionary=id2word, coherence='c_v')\n",
653 "coherence_lda = coherence_model_lda.get_coherence()\n",
654 "print('\\nCoherence Score ', coherence_lda)"
655 ]
656 },
657 {
658 "cell_type": "code",
659 "execution_count": null,
660 "id": "cece5b6e",
661 "metadata": {},
662 "outputs": [],
663 "source": [
664 "coherence=[]\n",
665 "for i in range(2,16):\n",
666 " lda= gensim.models.ldamodel.LdaModel(corpus=corpus,\n",
667 " num_topics=i,\n",
668 " id2word=id2word,\n",
669 " chunksize=200,\n",
670 " passes=10,\n",
671 " update_every=1,\n",
672 " alpha='auto',\n",
673 " random_state=100)\n",
674 " coherence_model_lda = CoherenceModel(model=lda, texts=data_bi_tri, dictionary=id2word, coherence='c_v')\n",
675 " coherence_lda = coherence_model_lda.get_coherence()\n",
676 " print('\\nCoherence Score for '+str(i)+'topics: ', coherence_lda)\n",
677 " coherence.append(coherence_lda)"
678 ]
679 },
680 {
681 "cell_type": "code",
682 "execution_count": null,
683 "id": "61ffcf9a",
684 "metadata": {},
685 "outputs": [],
686 "source": [
687 "import matplotlib.pyplot as plt\n",
688 "\n",
689 "plt.plot(coherence)"
690 ]
691 },
692 {
693 "cell_type": "code",
694 "execution_count": null,
695 "id": "911d8501",
696 "metadata": {},
697 "outputs": [],
698 "source": []
699 }
700 ],
701 "metadata": {
702 "kernelspec": {
703 "display_name": "Python 3",
704 "language": "python",
705 "name": "python3"
706 },
707 "language_info": {
708 "codemirror_mode": {
709 "name": "ipython",
710 "version": 3
711 },
712 "file_extension": ".py",
713 "mimetype": "text/x-python",
714 "name": "python",
715 "nbconvert_exporter": "python",
716 "pygments_lexer": "ipython3",
717 "version": "3.8.8"
718 }
719 },
720 "nbformat": 4,
721 "nbformat_minor": 5
722 }
723