Format: application/octet-stream
1 {
2 "cells": [
3 {
4 "cell_type": "code",
5 "execution_count": 1,
6 "id": "f69f9c65",
7 "metadata": {},
8 "outputs": [],
9 "source": [
10 "import pandas as pd\n",
11 "import spacy \n",
12 "import numpy as np\n",
13 "nlp= spacy.load('en_core_web_md')"
14 ]
15 },
16 {
17 "cell_type": "code",
18 "execution_count": 2,
19 "id": "a0f69bfe",
20 "metadata": {},
21 "outputs": [],
22 "source": [
23 "movies= pd.read_csv('data/synopsis allowed pos.tsv', sep='\\t').dropna(axis=0)"
24 ]
25 },
26 {
27 "cell_type": "code",
28 "execution_count": 3,
29 "id": "25a3540a",
30 "metadata": {},
31 "outputs": [
32 {
33 "data": {
34 "text/html": [
35 "<div>\n",
36 "<style scoped>\n",
37 " .dataframe tbody tr th:only-of-type {\n",
38 " vertical-align: middle;\n",
39 " }\n",
40 "\n",
41 " .dataframe tbody tr th {\n",
42 " vertical-align: top;\n",
43 " }\n",
44 "\n",
45 " .dataframe thead th {\n",
46 " text-align: right;\n",
47 " }\n",
48 "</style>\n",
49 "<table border=\"1\" class=\"dataframe\">\n",
50 " <thead>\n",
51 " <tr style=\"text-align: right;\">\n",
52 " <th></th>\n",
53 " <th>titleId</th>\n",
54 " <th>averageRating</th>\n",
55 " <th>numVotes</th>\n",
56 " <th>title</th>\n",
57 " <th>startYear</th>\n",
58 " <th>runtimeMinutes</th>\n",
59 " <th>genres</th>\n",
60 " <th>synopsis</th>\n",
61 " <th>synopsis_allowed_pos</th>\n",
62 " </tr>\n",
63 " </thead>\n",
64 " <tbody>\n",
65 " <tr>\n",
66 " <th>0</th>\n",
67 " <td>tt0111161</td>\n",
68 " <td>9.3</td>\n",
69 " <td>2405502</td>\n",
70 " <td>The Shawshank Redemption</td>\n",
71 " <td>1994</td>\n",
72 " <td>142</td>\n",
73 " <td>Drama</td>\n",
74 " <td>In 1947, Andy Dufresne (Tim Robbins), a banker...</td>\n",
75 " <td>banker wife lover golf state death penalty con...</td>\n",
76 " </tr>\n",
77 " <tr>\n",
78 " <th>1</th>\n",
79 " <td>tt0468569</td>\n",
80 " <td>9.0</td>\n",
81 " <td>2365212</td>\n",
82 " <td>The Dark Knight</td>\n",
83 " <td>2008</td>\n",
84 " <td>152</td>\n",
85 " <td>Action,Crime,Drama</td>\n",
86 " <td>The movie begins with a gang of men with clown...</td>\n",
87 " <td>movie gang man clown mask bank mob large porti...</td>\n",
88 " </tr>\n",
89 " <tr>\n",
90 " <th>2</th>\n",
91 " <td>tt1375666</td>\n",
92 " <td>8.8</td>\n",
93 " <td>2122723</td>\n",
94 " <td>Inception</td>\n",
95 " <td>2010</td>\n",
96 " <td>148</td>\n",
97 " <td>Action,Adventure,Sci-Fi</td>\n",
98 " <td>A young man, exhausted and delirious, washes u...</td>\n",
99 " <td>young man exhausted delirious beach see young ...</td>\n",
100 " </tr>\n",
101 " <tr>\n",
102 " <th>3</th>\n",
103 " <td>tt0137523</td>\n",
104 " <td>8.8</td>\n",
105 " <td>1899050</td>\n",
106 " <td>Fight Club</td>\n",
107 " <td>1999</td>\n",
108 " <td>139</td>\n",
109 " <td>Drama</td>\n",
110 " <td>We back out of the webbing of neurons and brai...</td>\n",
111 " <td>webbing neuron brain cell title credit pore sw...</td>\n",
112 " </tr>\n",
113 " <tr>\n",
114 " <th>4</th>\n",
115 " <td>tt0110912</td>\n",
116 " <td>8.9</td>\n",
117 " <td>1868911</td>\n",
118 " <td>Pulp Fiction</td>\n",
119 " <td>1994</td>\n",
120 " <td>154</td>\n",
121 " <td>Crime,Drama</td>\n",
122 " <td>Late one morning in the Hawthorne Grill, a res...</td>\n",
123 " <td>morning restaurant young couple discuss pro co...</td>\n",
124 " </tr>\n",
125 " <tr>\n",
126 " <th>...</th>\n",
127 " <td>...</td>\n",
128 " <td>...</td>\n",
129 " <td>...</td>\n",
130 " <td>...</td>\n",
131 " <td>...</td>\n",
132 " <td>...</td>\n",
133 " <td>...</td>\n",
134 " <td>...</td>\n",
135 " <td>...</td>\n",
136 " </tr>\n",
137 " <tr>\n",
138 " <th>7135</th>\n",
139 " <td>tt4587656</td>\n",
140 " <td>6.3</td>\n",
141 " <td>7088</td>\n",
142 " <td>The Star</td>\n",
143 " <td>2017</td>\n",
144 " <td>86</td>\n",
145 " <td>Adventure,Animation,Comedy</td>\n",
146 " <td>The film opens in Nazareth where Mary (voice o...</td>\n",
147 " <td>film voice angel voice bear son pygmy mouse vo...</td>\n",
148 " </tr>\n",
149 " <tr>\n",
150 " <th>7136</th>\n",
151 " <td>tt0035238</td>\n",
152 " <td>8.0</td>\n",
153 " <td>7087</td>\n",
154 " <td>Random Harvest</td>\n",
155 " <td>1942</td>\n",
156 " <td>126</td>\n",
157 " <td>Adventure,Drama,Family</td>\n",
158 " <td>A World War I veteran is languishing in an Eng...</td>\n",
159 " <td>english asylum shell shock amnesia remember na...</td>\n",
160 " </tr>\n",
161 " <tr>\n",
162 " <th>7137</th>\n",
163 " <td>tt0075765</td>\n",
164 " <td>6.8</td>\n",
165 " <td>7087</td>\n",
166 " <td>Black Sunday</td>\n",
167 " <td>1977</td>\n",
168 " <td>143</td>\n",
169 " <td>Adventure,Crime,Drama</td>\n",
170 " <td>On November 12, 1975, Dahlia Iyad (Marthe Kell...</td>\n",
171 " <td>airport taxi town narrow alley bazaar back sea...</td>\n",
172 " </tr>\n",
173 " <tr>\n",
174 " <th>7138</th>\n",
175 " <td>tt0087197</td>\n",
176 " <td>6.5</td>\n",
177 " <td>7087</td>\n",
178 " <td>Electric Dreams</td>\n",
179 " <td>1984</td>\n",
180 " <td>95</td>\n",
181 " <td>Comedy,Drama,Music</td>\n",
182 " <td>Lonely architect Miles (Von Dohlen) purchases ...</td>\n",
183 " <td>lonely architect fancy new cone personal compu...</td>\n",
184 " </tr>\n",
185 " <tr>\n",
186 " <th>7139</th>\n",
187 " <td>tt0058725</td>\n",
188 " <td>6.4</td>\n",
189 " <td>7084</td>\n",
190 " <td>Viva Las Vegas</td>\n",
191 " <td>1964</td>\n",
192 " <td>85</td>\n",
193 " <td>Comedy,Musical</td>\n",
194 " <td>Lucky Jackson (Elvis Presley) goes to Las Vega...</td>\n",
195 " <td>participate city first annual race car need ne...</td>\n",
196 " </tr>\n",
197 " </tbody>\n",
198 "</table>\n",
199 "<p>7138 rows × 9 columns</p>\n",
200 "</div>"
201 ],
202 "text/plain": [
203 " titleId averageRating numVotes title startYear \\\n",
204 "0 tt0111161 9.3 2405502 The Shawshank Redemption 1994 \n",
205 "1 tt0468569 9.0 2365212 The Dark Knight 2008 \n",
206 "2 tt1375666 8.8 2122723 Inception 2010 \n",
207 "3 tt0137523 8.8 1899050 Fight Club 1999 \n",
208 "4 tt0110912 8.9 1868911 Pulp Fiction 1994 \n",
209 "... ... ... ... ... ... \n",
210 "7135 tt4587656 6.3 7088 The Star 2017 \n",
211 "7136 tt0035238 8.0 7087 Random Harvest 1942 \n",
212 "7137 tt0075765 6.8 7087 Black Sunday 1977 \n",
213 "7138 tt0087197 6.5 7087 Electric Dreams 1984 \n",
214 "7139 tt0058725 6.4 7084 Viva Las Vegas 1964 \n",
215 "\n",
216 " runtimeMinutes genres \\\n",
217 "0 142 Drama \n",
218 "1 152 Action,Crime,Drama \n",
219 "2 148 Action,Adventure,Sci-Fi \n",
220 "3 139 Drama \n",
221 "4 154 Crime,Drama \n",
222 "... ... ... \n",
223 "7135 86 Adventure,Animation,Comedy \n",
224 "7136 126 Adventure,Drama,Family \n",
225 "7137 143 Adventure,Crime,Drama \n",
226 "7138 95 Comedy,Drama,Music \n",
227 "7139 85 Comedy,Musical \n",
228 "\n",
229 " synopsis \\\n",
230 "0 In 1947, Andy Dufresne (Tim Robbins), a banker... \n",
231 "1 The movie begins with a gang of men with clown... \n",
232 "2 A young man, exhausted and delirious, washes u... \n",
233 "3 We back out of the webbing of neurons and brai... \n",
234 "4 Late one morning in the Hawthorne Grill, a res... \n",
235 "... ... \n",
236 "7135 The film opens in Nazareth where Mary (voice o... \n",
237 "7136 A World War I veteran is languishing in an Eng... \n",
238 "7137 On November 12, 1975, Dahlia Iyad (Marthe Kell... \n",
239 "7138 Lonely architect Miles (Von Dohlen) purchases ... \n",
240 "7139 Lucky Jackson (Elvis Presley) goes to Las Vega... \n",
241 "\n",
242 " synopsis_allowed_pos \n",
243 "0 banker wife lover golf state death penalty con... \n",
244 "1 movie gang man clown mask bank mob large porti... \n",
245 "2 young man exhausted delirious beach see young ... \n",
246 "3 webbing neuron brain cell title credit pore sw... \n",
247 "4 morning restaurant young couple discuss pro co... \n",
248 "... ... \n",
249 "7135 film voice angel voice bear son pygmy mouse vo... \n",
250 "7136 english asylum shell shock amnesia remember na... \n",
251 "7137 airport taxi town narrow alley bazaar back sea... \n",
252 "7138 lonely architect fancy new cone personal compu... \n",
253 "7139 participate city first annual race car need ne... \n",
254 "\n",
255 "[7138 rows x 9 columns]"
256 ]
257 },
258 "execution_count": 3,
259 "metadata": {},
260 "output_type": "execute_result"
261 }
262 ],
263 "source": [
264 "movies"
265 ]
266 },
267 {
268 "cell_type": "code",
269 "execution_count": null,
270 "id": "3712a0dd",
271 "metadata": {},
272 "outputs": [],
273 "source": []
274 },
275 {
276 "cell_type": "code",
277 "execution_count": null,
278 "id": "06481335",
279 "metadata": {},
280 "outputs": [],
281 "source": []
282 },
283 {
284 "cell_type": "code",
285 "execution_count": null,
286 "id": "3afd35c1",
287 "metadata": {},
288 "outputs": [],
289 "source": []
290 },
291 {
292 "cell_type": "code",
293 "execution_count": 4,
294 "id": "3616f6a2",
295 "metadata": {},
296 "outputs": [],
297 "source": [
298 "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
299 "from nltk.tokenize import word_tokenize\n",
300 "from nltk.stem import WordNetLemmatizer\n",
301 "from nltk.corpus import stopwords\n",
302 "from nltk import pos_tag, pos_tag_sents"
303 ]
304 },
305 {
306 "cell_type": "code",
307 "execution_count": 5,
308 "id": "00701c21",
309 "metadata": {},
310 "outputs": [],
311 "source": [
312 "def pre_process(text, allowed_pos=['NNS','NN',\"ADJ\",'JJ','JJR', 'VB','RBR']):\n",
313 " words= nlp(' '.join(text.split('.')))\n",
314 " words= [ token.lemma_ for token in words if token.tag_ in allowed_pos or token.text=='<__SPLIT__>']\n",
315 " return ' '.join(words)"
316 ]
317 },
318 {
319 "cell_type": "code",
320 "execution_count": null,
321 "id": "6318de53",
322 "metadata": {},
323 "outputs": [],
324 "source": []
325 },
326 {
327 "cell_type": "code",
328 "execution_count": null,
329 "id": "67b1fbd4",
330 "metadata": {},
331 "outputs": [],
332 "source": []
333 },
334 {
335 "cell_type": "code",
336 "execution_count": null,
337 "id": "daee27c3",
338 "metadata": {},
339 "outputs": [],
340 "source": []
341 },
342 {
343 "cell_type": "code",
344 "execution_count": null,
345 "id": "d4293b53",
346 "metadata": {},
347 "outputs": [],
348 "source": []
349 },
350 {
351 "cell_type": "code",
352 "execution_count": 53,
353 "id": "d643149a",
354 "metadata": {},
355 "outputs": [],
356 "source": [
357 "def vectorize(docs,n, max_df=0.5):\n",
358 " cv= CountVectorizer(max_df=max_df, max_features= 200000, ngram_range=n, lowercase= True, stop_words=stopwords)\n",
359 " word_count_vector= cv.fit_transform(docs)\n",
360 " features=cv.get_feature_names()\n",
361 " return word_count_vector, features, cv"
362 ]
363 },
364 {
365 "cell_type": "code",
366 "execution_count": null,
367 "id": "4f01462c",
368 "metadata": {},
369 "outputs": [],
370 "source": []
371 },
372 {
373 "cell_type": "code",
374 "execution_count": 7,
375 "id": "87854f40",
376 "metadata": {},
377 "outputs": [],
378 "source": [
379 "def sort_matrix(coo_matrix):\n",
380 " tuples = zip(coo_matrix.col, coo_matrix.data)\n",
381 " return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)"
382 ]
383 },
384 {
385 "cell_type": "code",
386 "execution_count": 8,
387 "id": "15f4fb9d",
388 "metadata": {},
389 "outputs": [],
390 "source": [
391 "def extract_topn_from_vector(feature_names, sorted_items, topn, allowed_pos=['NNS','NN',\"ADJ\",'JJ','JJR', 'VB','RBR']):\n",
392 " \"\"\"get the feature names and tf-idf score of top n items\"\"\"\n",
393 " \n",
394 " #use only topn items from vector\n",
395 " #sorted_items = sorted_items[:topn]\n",
396 " sorted_items=sorted_items[:topn+1]\n",
397 " score_vals = []\n",
398 " feature_vals = []\n",
399 "\n",
400 " for idx, score in sorted_items:\n",
401 " fname = feature_names[idx]\n",
402 " #keep track of feature name and its corresponding score\n",
403 " score_vals.append(score)\n",
404 " feature_vals.append(feature_names[idx])\n",
405 "\n",
406 " #create a tuples of feature,score\n",
407 " #results = zip(feature_vals,score_vals)\n",
408 " results= {}\n",
409 " for idx in range(len(feature_vals)):\n",
410 " results[feature_vals[idx]]=score_vals[idx]\n",
411 " \n",
412 " return results"
413 ]
414 },
415 {
416 "cell_type": "code",
417 "execution_count": 9,
418 "id": "e3fd0504",
419 "metadata": {},
420 "outputs": [],
421 "source": [
422 "def get_features(text, features_names, n ,cv, word_count_vector):\n",
423 " #print(movies.title[movies.synopsis_allowed_pos==text].values[0]) #title\n",
424 " #print(movies.title[movies.synopsis==text].values[0]) #title\n",
425 " tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True)\n",
426 " tfidf_transformer.fit(word_count_vector)\n",
427 " tf_idf_vector=tfidf_transformer.transform(cv.transform([text]))\n",
428 " \n",
429 " sorted_items=sort_matrix(tf_idf_vector.tocoo())\n",
430 " \n",
431 " keywords=extract_topn_from_vector(features_names,sorted_items,n)\n",
432 " return keywords"
433 ]
434 },
435 {
436 "cell_type": "code",
437 "execution_count": null,
438 "id": "d16cd55a",
439 "metadata": {},
440 "outputs": [],
441 "source": []
442 },
443 {
444 "cell_type": "code",
445 "execution_count": 10,
446 "id": "3508065b",
447 "metadata": {},
448 "outputs": [],
449 "source": [
450 "stopwords=stopwords.words('english')"
451 ]
452 },
453 {
454 "cell_type": "code",
455 "execution_count": null,
456 "id": "fe4175bf",
457 "metadata": {},
458 "outputs": [],
459 "source": []
460 },
461 {
462 "cell_type": "code",
463 "execution_count": null,
464 "id": "956907f5",
465 "metadata": {},
466 "outputs": [],
467 "source": []
468 },
469 {
470 "cell_type": "code",
471 "execution_count": null,
472 "id": "f05d1e26",
473 "metadata": {},
474 "outputs": [],
475 "source": []
476 },
477 {
478 "cell_type": "code",
479 "execution_count": 11,
480 "id": "a44e1795",
481 "metadata": {},
482 "outputs": [],
483 "source": [
484 "m=movies.copy()"
485 ]
486 },
487 {
488 "cell_type": "code",
489 "execution_count": 50,
490 "id": "b0cbc8e3",
491 "metadata": {},
492 "outputs": [],
493 "source": [
494 "def apply(text, n=50):\n",
495 " a=get_features(text, features2, n, cv2, word_count_vector2)\n",
496 " f= ','.join(list(a.keys()))\n",
497 " a=','.join([str(i) for i in list(a.values()) ])\n",
498 " return f, a"
499 ]
500 },
501 {
502 "cell_type": "code",
503 "execution_count": 77,
504 "id": "52c1a2b1",
505 "metadata": {},
506 "outputs": [],
507 "source": [
508 "word_count_vector2, features2, cv2= vectorize(movies.synopsis,(5,5))\n",
509 "#x= m.synopsis_allowed_pos.apply(apply)"
510 ]
511 },
512 {
513 "cell_type": "code",
514 "execution_count": 78,
515 "id": "ceee2eb8",
516 "metadata": {
517 "scrolled": true
518 },
519 "outputs": [
520 {
521 "name": "stdout",
522 "output_type": "stream",
523 "text": [
524 "['top continues spin wobbles bit', 'pressure robert using browning leverage', 'pressure robert combination tells series', 'pressed carpet saito begins laugh', 'presents grandpa grandmother hangs arthur', 'presence cobb gets joins ariadne', 'prepares next weeks arthur shows', 'prepares impersonation browning cobb arthur', 'prepare enter second level dream', 'practically non existent team angered', 'powerful sedative show effectiveness yusuf', 'postlethwaite slowly decaying illness eames', 'possible complete job team decides', 'possible ariadne confronts cobb control', 'possessed radical notions stranger looks', 'positive really except cobb hopes', 'positive idea trump negative one', 'positive ending cynics would see', 'pose smiles wryly apartment cobb', 'portable automated somnacin intravenous device', 'porch ariadne finds pushes improvised', 'polyester thus comes conclusion still', 'points woman nearby asking cobb', 'points someone better introduces cobb', 'points cabinet robert finds paper', 'pointing two men bar cobb', 'pointing temple top spins falls', 'pocketed arthur conversation starter drugs', 'plummets mid air loss gravity', 'plays music cue time running', 'plays music arthur headphones warn', 'plays faintly within dream enough', 'playing sand passes armed guard', 'planting idea someone mind rather', 'planting idea someone head guarantee', 'planted idea target mind robert', 'plans wants full verification success', 'plans ariadne finds cobb dreaming', 'plane lands los angeles cobb', 'plane access one evening day', 'plan uses phone wires tie', 'plaguing ever since tells mal', 'places headphones nash ears plays', 'placed within safe lying side', 'place team forger imitator eames', 'place robert gone limbo ariadne', 'place real home children cobb', 'pillow gun grabs arthur subdued', 'pillow arthur tells nash wake', 'piece totem wanting much time', 'picks shard glass cobb arrives']\n"
525 ]
526 }
527 ],
528 "source": [
529 "print(apply(m.synopsis[2],50)[0].split(','))"
530 ]
531 },
532 {
533 "cell_type": "code",
534 "execution_count": null,
535 "id": "3e3f90c8",
536 "metadata": {},
537 "outputs": [],
538 "source": []
539 },
540 {
541 "cell_type": "code",
542 "execution_count": 67,
543 "id": "ed17e2e3",
544 "metadata": {},
545 "outputs": [],
546 "source": [
547 "uni= ['dream', 'limbo', 'team', 'subconscious', 'projection', 'kick', 'top', 'level', 'father', 'mind', 'room', 'wake', 'inception', 'van', 'safe', 'job', 'brass', 'totem', 'gravity', 'folder', 'fort', 'manila', 'gun', 'elevator', 'reality', 'sedative', 'see', 'real', 'warehouse', 'child', 'mountainside', 'create', 'moment', 'interpretation', 'workshop', 'table', 'bridge', 'someone', 'carpet', 'asleep', 'impossible', 'guard', 'mob', 'subjective', 'idea', 'architect', 'spin', 'extractor', 'access', 'combination', 'train']"
548 ]
549 },
550 {
551 "cell_type": "code",
552 "execution_count": 64,
553 "id": "0894ef27",
554 "metadata": {},
555 "outputs": [],
556 "source": [
557 "bi=['brass top', 'level dream', 'top spin', 'manila folder', 'dream van', 'dream projection', 'team dream', 'father company', 'subjective interpretation', 'stay limbo', 'split father', 'spinning top', 'spin bit', 'seaside palace', 'elderly japanese', 'dream level', 'third level', 'information file', 'door elated', 'kick time', 'elated see', 'perform job', 'man stranger', 'complete job', 'idea someone', 'dream state', 'way wake', 'time clock', 'room safe', 'child back', 'japanese man', 'father last', 'armed guard', 'car train', 'woman take', 'dream dream', 'hotel room', 'old man', 'rest team', 'last word', 'real world', 'last moment', 'gun head', 'back door', 'warehouse angry', 'view person', 'taxi pick', 'tank van', 'take subject', 'success mask', 'stranger arrival']"
558 ]
559 },
560 {
561 "cell_type": "code",
562 "execution_count": 70,
563 "id": "426a4de0",
564 "metadata": {},
565 "outputs": [],
566 "source": [
567 "tri=['top spin bit', 'split father company', 'man brass top', 'elderly japanese man', 'door elated see', 'child back door', 'back door elated', 'father last word', 'roof fly lie', 'rock fort main', 'risk armed force', 'rioting mob near', 'rioter street dream', 'ring hear child', 'rest film everyone', 'responsible kidnapping second', 'resolve use method', 'resolve relation go', 'resist pure creation', 'require use dream', 'remember safe alternate', 'remember hotel room', 'relief gun phone', 'release guilt death', 'relationship positive idea', 'relation go child', 'reject true inspiration', 'regard door helicopter', 'reasoning job ceo', 'powerful sedative show', 'possible control subconscious', 'possible complete job', 'positive idea trump', 'positive hope director', 'positive ending cynic', 'pose apartment brass', 'porch kick life', 'polyester conclusion asleep', 'pocketed conversation starter', 'plane custom security', 'plane access evening', 'plan workshop curious', 'plan phone wire', 'plan full verification', 'place team forger', 'place real home', 'place limbo limbo', 'pillow wake kick', 'pillow gun information', 'piece totem much', 'pick pocketed conversation']"
568 ]
569 },
570 {
571 "cell_type": "code",
572 "execution_count": null,
573 "id": "f463b8d5",
574 "metadata": {},
575 "outputs": [],
576 "source": []
577 },
578 {
579 "cell_type": "code",
580 "execution_count": null,
581 "id": "98f00096",
582 "metadata": {},
583 "outputs": [],
584 "source": []
585 },
586 {
587 "cell_type": "code",
588 "execution_count": 71,
589 "id": "79336082",
590 "metadata": {},
591 "outputs": [
592 {
593 "name": "stdout",
594 "output_type": "stream",
595 "text": [
596 "['dream', 'limbo', 'team', 'subconscious', 'projection', 'kick', 'top', 'level', 'father', 'mind', 'room', 'wake', 'inception', 'van', 'safe', 'job', 'brass', 'totem', 'gravity', 'folder'] \n",
597 " ['brass top', 'level dream', 'top spin', 'manila folder', 'dream van', 'dream projection', 'team dream', 'father company', 'subjective interpretation', 'stay limbo', 'split father', 'spinning top', 'spin bit', 'seaside palace', 'elderly japanese'] \n",
598 " ['top spin bit', 'split father company', 'man brass top', 'elderly japanese man', 'door elated see', 'child back door', 'back door elated', 'father last word', 'roof fly lie', 'rock fort main', 'risk armed force', 'rioting mob near', 'rioter street dream', 'ring hear child', 'rest film everyone']\n"
599 ]
600 }
601 ],
602 "source": [
603 "print(uni[:20],'\\n', bi[:15],'\\n', tri[:15])"
604 ]
605 },
606 {
607 "cell_type": "code",
608 "execution_count": null,
609 "id": "19a92fe3",
610 "metadata": {},
611 "outputs": [],
612 "source": []
613 },
614 {
615 "cell_type": "code",
616 "execution_count": null,
617 "id": "47d30863",
618 "metadata": {},
619 "outputs": [],
620 "source": []
621 },
622 {
623 "cell_type": "code",
624 "execution_count": null,
625 "id": "ad31c89f",
626 "metadata": {},
627 "outputs": [],
628 "source": []
629 },
630 {
631 "cell_type": "code",
632 "execution_count": null,
633 "id": "316370eb",
634 "metadata": {},
635 "outputs": [],
636 "source": []
637 },
638 {
639 "cell_type": "code",
640 "execution_count": null,
641 "id": "bbe598c1",
642 "metadata": {},
643 "outputs": [],
644 "source": []
645 },
646 {
647 "cell_type": "code",
648 "execution_count": null,
649 "id": "6330c6a4",
650 "metadata": {},
651 "outputs": [],
652 "source": []
653 },
654 {
655 "cell_type": "code",
656 "execution_count": null,
657 "id": "1e5241b9",
658 "metadata": {},
659 "outputs": [],
660 "source": []
661 },
662 {
663 "cell_type": "code",
664 "execution_count": null,
665 "id": "16af6fa5",
666 "metadata": {},
667 "outputs": [],
668 "source": []
669 },
670 {
671 "cell_type": "code",
672 "execution_count": 15,
673 "id": "741f01f4",
674 "metadata": {},
675 "outputs": [],
676 "source": [
677 "keywords=[]\n",
678 "scores=[]\n",
679 "for i in x:\n",
680 " keywords.append(i[0])\n",
681 " scores.append(i[1])"
682 ]
683 },
684 {
685 "cell_type": "code",
686 "execution_count": null,
687 "id": "718e4fb1",
688 "metadata": {},
689 "outputs": [],
690 "source": []
691 },
692 {
693 "cell_type": "code",
694 "execution_count": 16,
695 "id": "78ee9591",
696 "metadata": {},
697 "outputs": [],
698 "source": [
699 "m['keywords']= keywords\n",
700 "m['scores']= scores"
701 ]
702 },
703 {
704 "cell_type": "code",
705 "execution_count": 17,
706 "id": "49f49f2c",
707 "metadata": {},
708 "outputs": [
709 {
710 "data": {
711 "text/html": [
712 "<div>\n",
713 "<style scoped>\n",
714 " .dataframe tbody tr th:only-of-type {\n",
715 " vertical-align: middle;\n",
716 " }\n",
717 "\n",
718 " .dataframe tbody tr th {\n",
719 " vertical-align: top;\n",
720 " }\n",
721 "\n",
722 " .dataframe thead th {\n",
723 " text-align: right;\n",
724 " }\n",
725 "</style>\n",
726 "<table border=\"1\" class=\"dataframe\">\n",
727 " <thead>\n",
728 " <tr style=\"text-align: right;\">\n",
729 " <th></th>\n",
730 " <th>titleId</th>\n",
731 " <th>averageRating</th>\n",
732 " <th>numVotes</th>\n",
733 " <th>title</th>\n",
734 " <th>startYear</th>\n",
735 " <th>runtimeMinutes</th>\n",
736 " <th>genres</th>\n",
737 " <th>synopsis</th>\n",
738 " <th>synopsis_allowed_pos</th>\n",
739 " <th>keywords</th>\n",
740 " <th>scores</th>\n",
741 " </tr>\n",
742 " </thead>\n",
743 " <tbody>\n",
744 " <tr>\n",
745 " <th>0</th>\n",
746 " <td>tt0111161</td>\n",
747 " <td>9.3</td>\n",
748 " <td>2405502</td>\n",
749 " <td>The Shawshank Redemption</td>\n",
750 " <td>1994</td>\n",
751 " <td>142</td>\n",
752 " <td>Drama</td>\n",
753 " <td>In 1947, Andy Dufresne (Tim Robbins), a banker...</td>\n",
754 " <td>banker wife lover golf state death penalty con...</td>\n",
755 " <td>prison,warden,inmate,solitary,parole,cell,rock...</td>\n",
756 " <td>0.42707281600197916,0.3342030929304552,0.24341...</td>\n",
757 " </tr>\n",
758 " <tr>\n",
759 " <th>1</th>\n",
760 " <td>tt0468569</td>\n",
761 " <td>9.0</td>\n",
762 " <td>2365212</td>\n",
763 " <td>The Dark Knight</td>\n",
764 " <td>2008</td>\n",
765 " <td>152</td>\n",
766 " <td>Action,Crime,Drama</td>\n",
767 " <td>The movie begins with a gang of men with clown...</td>\n",
768 " <td>movie gang man clown mask bank mob large porti...</td>\n",
769 " <td>mob,coin,clown,ferry,member,building,cell,deto...</td>\n",
770 " <td>0.307883706188824,0.30332210275656385,0.225254...</td>\n",
771 " </tr>\n",
772 " <tr>\n",
773 " <th>2</th>\n",
774 " <td>tt1375666</td>\n",
775 " <td>8.8</td>\n",
776 " <td>2122723</td>\n",
777 " <td>Inception</td>\n",
778 " <td>2010</td>\n",
779 " <td>148</td>\n",
780 " <td>Action,Adventure,Sci-Fi</td>\n",
781 " <td>A young man, exhausted and delirious, washes u...</td>\n",
782 " <td>young man exhausted delirious beach see young ...</td>\n",
783 " <td>dream,limbo,team,subconscious,projection,kick,...</td>\n",
784 " <td>0.5295977467942155,0.2829083379546759,0.228430...</td>\n",
785 " </tr>\n",
786 " <tr>\n",
787 " <th>3</th>\n",
788 " <td>tt0137523</td>\n",
789 " <td>8.8</td>\n",
790 " <td>1899050</td>\n",
791 " <td>Fight Club</td>\n",
792 " <td>1999</td>\n",
793 " <td>139</td>\n",
794 " <td>Drama</td>\n",
795 " <td>We back out of the webbing of neurons and brai...</td>\n",
796 " <td>webbing neuron brain cell title credit pore sw...</td>\n",
797 " <td>narrator,soap,member,fight,club,support group,...</td>\n",
798 " <td>0.9380025873693085,0.08926369620736493,0.07152...</td>\n",
799 " </tr>\n",
800 " <tr>\n",
801 " <th>4</th>\n",
802 " <td>tt0110912</td>\n",
803 " <td>8.9</td>\n",
804 " <td>1868911</td>\n",
805 " <td>Pulp Fiction</td>\n",
806 " <td>1994</td>\n",
807 " <td>154</td>\n",
808 " <td>Crime,Drama</td>\n",
809 " <td>Late one morning in the Hawthorne Grill, a res...</td>\n",
810 " <td>morning restaurant young couple discuss pro co...</td>\n",
811 " <td>watch,pawn shop,apartment,strip,strip club,paw...</td>\n",
812 " <td>0.20000712122969372,0.1685305060289841,0.16701...</td>\n",
813 " </tr>\n",
814 " <tr>\n",
815 " <th>...</th>\n",
816 " <td>...</td>\n",
817 " <td>...</td>\n",
818 " <td>...</td>\n",
819 " <td>...</td>\n",
820 " <td>...</td>\n",
821 " <td>...</td>\n",
822 " <td>...</td>\n",
823 " <td>...</td>\n",
824 " <td>...</td>\n",
825 " <td>...</td>\n",
826 " <td>...</td>\n",
827 " </tr>\n",
828 " <tr>\n",
829 " <th>7135</th>\n",
830 " <td>tt4587656</td>\n",
831 " <td>6.3</td>\n",
832 " <td>7088</td>\n",
833 " <td>The Star</td>\n",
834 " <td>2017</td>\n",
835 " <td>86</td>\n",
836 " <td>Adventure,Animation,Comedy</td>\n",
837 " <td>The film opens in Nazareth where Mary (voice o...</td>\n",
838 " <td>film voice angel voice bear son pygmy mouse vo...</td>\n",
839 " <td>donkey,voice,camel,dog,animal,voice voice,baby...</td>\n",
840 " <td>0.5929905562344081,0.4319514399740728,0.252430...</td>\n",
841 " </tr>\n",
842 " <tr>\n",
843 " <th>7136</th>\n",
844 " <td>tt0035238</td>\n",
845 " <td>8.0</td>\n",
846 " <td>7087</td>\n",
847 " <td>Random Harvest</td>\n",
848 " <td>1942</td>\n",
849 " <td>126</td>\n",
850 " <td>Adventure,Drama,Family</td>\n",
851 " <td>A World War I veteran is languishing in an Eng...</td>\n",
852 " <td>english asylum shell shock amnesia remember na...</td>\n",
853 " <td>memory,asylum,amnesia,marriage,town,married,lo...</td>\n",
854 " <td>0.4533770514366685,0.2975889560040371,0.232611...</td>\n",
855 " </tr>\n",
856 " <tr>\n",
857 " <th>7137</th>\n",
858 " <td>tt0075765</td>\n",
859 " <td>6.8</td>\n",
860 " <td>7087</td>\n",
861 " <td>Black Sunday</td>\n",
862 " <td>1977</td>\n",
863 " <td>143</td>\n",
864 " <td>Adventure,Crime,Drama</td>\n",
865 " <td>On November 12, 1975, Dahlia Iyad (Marthe Kell...</td>\n",
866 " <td>airport taxi town narrow alley bazaar back sea...</td>\n",
867 " <td>helicopter,stadium,israeli,compound,shrapnel,p...</td>\n",
868 " <td>0.28705447424790514,0.2343437207076858,0.22922...</td>\n",
869 " </tr>\n",
870 " <tr>\n",
871 " <th>7138</th>\n",
872 " <td>tt0087197</td>\n",
873 " <td>6.5</td>\n",
874 " <td>7087</td>\n",
875 " <td>Electric Dreams</td>\n",
876 " <td>1984</td>\n",
877 " <td>95</td>\n",
878 " <td>Comedy,Drama,Music</td>\n",
879 " <td>Lonely architect Miles (Von Dohlen) purchases ...</td>\n",
880 " <td>lonely architect fancy new cone personal compu...</td>\n",
881 " <td>computer,musical,talent,human,romance,course,m...</td>\n",
882 " <td>0.3230515332221356,0.31746347822102616,0.28653...</td>\n",
883 " </tr>\n",
884 " <tr>\n",
885 " <th>7139</th>\n",
886 " <td>tt0058725</td>\n",
887 " <td>6.4</td>\n",
888 " <td>7084</td>\n",
889 " <td>Viva Las Vegas</td>\n",
890 " <td>1964</td>\n",
891 " <td>85</td>\n",
892 " <td>Comedy,Musical</td>\n",
893 " <td>Lucky Jackson (Elvis Presley) goes to Las Vega...</td>\n",
894 " <td>participate city first annual race car need ne...</td>\n",
895 " <td>hotel,win,contest,affection,necessary,talent,e...</td>\n",
896 " <td>0.3411526810073388,0.24019391232097287,0.21528...</td>\n",
897 " </tr>\n",
898 " </tbody>\n",
899 "</table>\n",
900 "<p>7138 rows × 11 columns</p>\n",
901 "</div>"
902 ],
903 "text/plain": [
904 " titleId averageRating numVotes title startYear \\\n",
905 "0 tt0111161 9.3 2405502 The Shawshank Redemption 1994 \n",
906 "1 tt0468569 9.0 2365212 The Dark Knight 2008 \n",
907 "2 tt1375666 8.8 2122723 Inception 2010 \n",
908 "3 tt0137523 8.8 1899050 Fight Club 1999 \n",
909 "4 tt0110912 8.9 1868911 Pulp Fiction 1994 \n",
910 "... ... ... ... ... ... \n",
911 "7135 tt4587656 6.3 7088 The Star 2017 \n",
912 "7136 tt0035238 8.0 7087 Random Harvest 1942 \n",
913 "7137 tt0075765 6.8 7087 Black Sunday 1977 \n",
914 "7138 tt0087197 6.5 7087 Electric Dreams 1984 \n",
915 "7139 tt0058725 6.4 7084 Viva Las Vegas 1964 \n",
916 "\n",
917 " runtimeMinutes genres \\\n",
918 "0 142 Drama \n",
919 "1 152 Action,Crime,Drama \n",
920 "2 148 Action,Adventure,Sci-Fi \n",
921 "3 139 Drama \n",
922 "4 154 Crime,Drama \n",
923 "... ... ... \n",
924 "7135 86 Adventure,Animation,Comedy \n",
925 "7136 126 Adventure,Drama,Family \n",
926 "7137 143 Adventure,Crime,Drama \n",
927 "7138 95 Comedy,Drama,Music \n",
928 "7139 85 Comedy,Musical \n",
929 "\n",
930 " synopsis \\\n",
931 "0 In 1947, Andy Dufresne (Tim Robbins), a banker... \n",
932 "1 The movie begins with a gang of men with clown... \n",
933 "2 A young man, exhausted and delirious, washes u... \n",
934 "3 We back out of the webbing of neurons and brai... \n",
935 "4 Late one morning in the Hawthorne Grill, a res... \n",
936 "... ... \n",
937 "7135 The film opens in Nazareth where Mary (voice o... \n",
938 "7136 A World War I veteran is languishing in an Eng... \n",
939 "7137 On November 12, 1975, Dahlia Iyad (Marthe Kell... \n",
940 "7138 Lonely architect Miles (Von Dohlen) purchases ... \n",
941 "7139 Lucky Jackson (Elvis Presley) goes to Las Vega... \n",
942 "\n",
943 " synopsis_allowed_pos \\\n",
944 "0 banker wife lover golf state death penalty con... \n",
945 "1 movie gang man clown mask bank mob large porti... \n",
946 "2 young man exhausted delirious beach see young ... \n",
947 "3 webbing neuron brain cell title credit pore sw... \n",
948 "4 morning restaurant young couple discuss pro co... \n",
949 "... ... \n",
950 "7135 film voice angel voice bear son pygmy mouse vo... \n",
951 "7136 english asylum shell shock amnesia remember na... \n",
952 "7137 airport taxi town narrow alley bazaar back sea... \n",
953 "7138 lonely architect fancy new cone personal compu... \n",
954 "7139 participate city first annual race car need ne... \n",
955 "\n",
956 " keywords \\\n",
957 "0 prison,warden,inmate,solitary,parole,cell,rock... \n",
958 "1 mob,coin,clown,ferry,member,building,cell,deto... \n",
959 "2 dream,limbo,team,subconscious,projection,kick,... \n",
960 "3 narrator,soap,member,fight,club,support group,... \n",
961 "4 watch,pawn shop,apartment,strip,strip club,paw... \n",
962 "... ... \n",
963 "7135 donkey,voice,camel,dog,animal,voice voice,baby... \n",
964 "7136 memory,asylum,amnesia,marriage,town,married,lo... \n",
965 "7137 helicopter,stadium,israeli,compound,shrapnel,p... \n",
966 "7138 computer,musical,talent,human,romance,course,m... \n",
967 "7139 hotel,win,contest,affection,necessary,talent,e... \n",
968 "\n",
969 " scores \n",
970 "0 0.42707281600197916,0.3342030929304552,0.24341... \n",
971 "1 0.307883706188824,0.30332210275656385,0.225254... \n",
972 "2 0.5295977467942155,0.2829083379546759,0.228430... \n",
973 "3 0.9380025873693085,0.08926369620736493,0.07152... \n",
974 "4 0.20000712122969372,0.1685305060289841,0.16701... \n",
975 "... ... \n",
976 "7135 0.5929905562344081,0.4319514399740728,0.252430... \n",
977 "7136 0.4533770514366685,0.2975889560040371,0.232611... \n",
978 "7137 0.28705447424790514,0.2343437207076858,0.22922... \n",
979 "7138 0.3230515332221356,0.31746347822102616,0.28653... \n",
980 "7139 0.3411526810073388,0.24019391232097287,0.21528... \n",
981 "\n",
982 "[7138 rows x 11 columns]"
983 ]
984 },
985 "execution_count": 17,
986 "metadata": {},
987 "output_type": "execute_result"
988 }
989 ],
990 "source": [
991 "m"
992 ]
993 },
994 {
995 "cell_type": "code",
996 "execution_count": null,
997 "id": "730484d4",
998 "metadata": {},
999 "outputs": [],
1000 "source": []
1001 },
1002 {
1003 "cell_type": "code",
1004 "execution_count": 18,
1005 "id": "4f70cbb2",
1006 "metadata": {},
1007 "outputs": [
1008 {
1009 "data": {
1010 "text/plain": [
1011 "titleId tt0378072\n",
1012 "averageRating 7.0\n",
1013 "numVotes 13478\n",
1014 "title Hum Tum\n",
1015 "startYear 2004\n",
1016 "runtimeMinutes 143\n",
1017 "genres Comedy,Drama,Romance\n",
1018 "synopsis Life isn't always like the movies. Love isn't ...\n",
1019 "synopsis_allowed_pos life movie love first sight relationship many ...\n",
1020 "keywords man woman,boy girl,relationship,hate,man littl...\n",
1021 "scores 0.3640233568367877,0.3030211769721811,0.287276...\n",
1022 "Name: 5802, dtype: object"
1023 ]
1024 },
1025 "execution_count": 18,
1026 "metadata": {},
1027 "output_type": "execute_result"
1028 }
1029 ],
1030 "source": [
1031 "m.iloc[5801,:]"
1032 ]
1033 },
1034 {
1035 "cell_type": "code",
1036 "execution_count": 19,
1037 "id": "3405781a",
1038 "metadata": {},
1039 "outputs": [],
1040 "source": [
1041 "m.to_csv('Final2.0.tsv', sep='\\t', index= False)"
1042 ]
1043 }
1044 ],
1045 "metadata": {
1046 "kernelspec": {
1047 "display_name": "Python 3",
1048 "language": "python",
1049 "name": "python3"
1050 },
1051 "language_info": {
1052 "codemirror_mode": {
1053 "name": "ipython",
1054 "version": 3
1055 },
1056 "file_extension": ".py",
1057 "mimetype": "text/x-python",
1058 "name": "python",
1059 "nbconvert_exporter": "python",
1060 "pygments_lexer": "ipython3",
1061 "version": "3.8.8"
1062 }
1063 },
1064 "nbformat": 4,
1065 "nbformat_minor": 5
1066 }
1067