{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "2c1eae7c", "metadata": {}, "outputs": [], "source": [ "import yake\n", "import pandas as pd\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "id": "40846086", "metadata": {}, "outputs": [], "source": [ "df=pd.read_csv('synopsis allowed pos.tsv', sep='\\t')" ] }, { "cell_type": "code", "execution_count": 3, "id": "6e6a9e84", "metadata": {}, "outputs": [], "source": [ "df.dropna(axis=0,inplace=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "482bd3c2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 4, "id": "2d77cc9b", "metadata": {}, "outputs": [], "source": [ "texts= df.iloc[:,-1].values" ] }, { "cell_type": "code", "execution_count": null, "id": "00ddf446", "metadata": {}, "outputs": [], "source": [ "texts" ] }, { "cell_type": "code", "execution_count": 5, "id": "79dd3a85", "metadata": {}, "outputs": [], "source": [ "from sklearn.feature_extraction.text import CountVectorizer\n", "def get_top_n_words(corpus, n):\n", " vec = CountVectorizer().fit(corpus)\n", " bag_of_words = vec.transform(corpus)\n", " sum_words = bag_of_words.sum(axis=0) \n", " words_freq = sorted([(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()], key = lambda x: x[1], reverse=True)\n", " words=[w[0] for w in words_freq[:n] ]\n", " return words" ] }, { "cell_type": "code", "execution_count": 6, "id": "1983c768", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(6691, 10457, 8574.0)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def corpus_lens(texts):\n", " l=[]\n", " for text in texts[:2]:\n", " #print(text)\n", " l.append(len(text))\n", " return min(l) , max(l), round(np.mean(l),2)\n", "corpus_lens(texts)" ] }, { "cell_type": "code", "execution_count": null, "id": "97b487f8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 7, "id": "595eacf0", "metadata": {}, "outputs": [], "source": [ "top_50= get_top_n_words( texts, 50)" ] }, { "cell_type": "code", "execution_count": null, "id": "c97b95ae", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 51, "id": "2ef5dc79", "metadata": {}, "outputs": [], "source": [ "def keywords(texts, top=30, n_gram=1):\n", " text_keys=[]\n", " i=1\n", " getKey=yake.KeywordExtractor(top=2*top, n=n_gram)\n", " for i,text in enumerate(texts):\n", " print(i, text[:10])\n", " keys=getKey.extract_keywords(text)\n", " #keys= [k[0] for k in keys][:top] #use this with small corpus\n", " keys= [k[0] for k in keys if k[0] not in top_50][:top] #use this with big corpus\n", " keys=' '.join(keys)\n", " text_keys.append(keys)\n", " return text_keys" ] }, { "cell_type": "code", "execution_count": 52, "id": "7f39aa2c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0 banker wif\n", "1 movie gang\n", "2 young man \n", "3 webbing ne\n", "4 morning re\n", "5 film feath\n", "6 man horseb\n", "7 screen gre\n", "8 prologue u\n", "9 opening sc\n", "10 late summe\n", "11 group elde\n", "12 eulogy end\n", "13 season hig\n", "14 film flash\n", "15 unidentifi\n", "16 slave bunc\n", "17 victor for\n", "18 film story\n", "19 student tr\n", "20 chapter ti\n", "21 director h\n", "22 american f\n", "23 note parag\n", "24 relocation\n", "25 movie tv a\n", "26 shot sever\n", "27 voiceover \n", "28 sequel pre\n", "29 movie grou\n", "30 ferryboat \n", "31 parallel s\n", "32 complex st\n", "33 story swee\n", "34 year old f\n", "35 title logo\n", "36 treasure h\n", "37 earth wait\n", "38 hitman cle\n", "39 opening sc\n", "40 remember r\n", "41 film man c\n", "42 year old d\n", "43 year futur\n", "44 credit fil\n", "45 scene red \n", "46 deck ship \n", "47 story plac\n", "48 invade dea\n", "49 opening cr\n" ] } ], "source": [ "text_keys= keywords(texts[:50])" ] }, { "cell_type": "code", "execution_count": 53, "id": "83645ac6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['prison warden cell inmate rock guard state wall prisoner red parole job solitary library book hammer work remark poster place order office morning record hope story rest cold live stone',\n", " 'mob member coin phone give building face cell clown side boat ferry bank guard cop large location truck criminal city scene turn fight mayor goon clean card window detonator dirty',\n", " 'dream team top job limbo mind level safe kick projection real subconscious van moment wake guard idea reality table elevator brass train asleep minute water create warehouse bridge gravity folder',\n", " 'narrator fight member club group soap building meeting mouth support sleep bar basement phone burn insomnia kitchen boss bomb consumerism live rule sex fat back recruit list officer testicular cancer',\n", " 'watch apartment strip early gang club pawn shop morning bathroom small talk tyranny restaurant wallet diner briefcase miracle evil building gold member store valuable boxer fight hour suit town dinner',\n", " 'bus school stop ping pong feather boat run white book leg ability college shrimping war buy shrimp visit anti breeze foot story listener change narration attitude disbelief indifference rapt veneration',\n", " 'sister king husband brother horse wall daughter deserter queen bastard watch ride eye wedding top compound letter sex bit kind check snow ned wolf pup eager imp kingdom throne dwarf',\n", " 'phone world tank matrix human real ship crew building program helicopter call line wall chair black machine code floor power back free computer reality pill spoon hovercraft elevator part city',\n", " 'hobbit orc ring escape meet high finger army temptation destroy creature heir plan large elvish fire mountain travel band strong prologue birthday conquer land earth enslavement bearer powerful magical individual',\n", " 'city hobbit king army power sword attack ship force fight love great large palantír vision gondorian rest horse pyre peace opening scene river present mind mountain age creature west ride',\n", " 'business meeting wedding assassination bodyguard daughter role hospital dealing attempt request bed hitman war small murder button italian service trade associate favor return criminal scene consigliere sicilian offer political protection',\n", " 'black hole planet space brand equation video station ship farmhouse world daughter mission crew humanity dust gravity age book ghost cryosleep message bunker wormhole case datum great beacon centrifuge save',\n", " 'bane henchman tunnel pistol cop reactor officer city fire escape weapon truck bomb prisoner prison mercenary back hostage front driver end team mask board submachine rope guard cement hospital nuclear',\n", " 'meth lab gang cancer partner pay operation produce season provide drug desert business evidence henchman murder nazi neo treatment brother attempt distribute product street problem local dealer lawyer distributor distribution',\n", " 'war army battle ring wizard journey fortress defender attack wall captive horse hobbit forest gate rider arrow white fell beast free water loyal rocky rope creature lord service land ambush',\n", " 'detective apartment murder scene victim mill city picture place painting talk captain sliver crime spot lawyer pregnant force obese plastic wall morning horror work spaghetti stomach open case blood evidence',\n", " 'slave bounty white brother horse hunter sheriff estate fire mandingo town paper cart small marshal owner rifle bullet buy saloon plantation free intelligence legend black return identify act speak shoot',\n", " 'gladiator praetorian guard arena emperor popularity audience sword stiletto brother approval afterlife crowd win affection victor force attack roman victory barbarian join successor involvement power discover betrayal fight skill close',\n", " 'train bat emitter toxin microwave antidote city water flashback mask thug member drug officer prison fight parent fear gas cave street company rabbit blue flower building suit party close ship',\n", " 'starling victim file information deal real team basement case survivor ambulance phone call serial attempt patient face riddle profile daughter transfer exchange childhood address cell local memory lamb building guard',\n", " 'soldier german cinema nitrate auditorium american reel box chapter officer tavern screen french end deal back line fourth radio operator swastika finger truck war premiere accent fire movie forehead explosive',\n", " 'portal helicarrier control weapon board group banner team carrier attack fury world cell scepter earth council aircraft mind assignment call god agent hero capability work office building credit fight ship',\n", " 'german squad soldier unit tank machine bridge fire brother beach letter position side grave bunker cover mission sniper small captain troop bomb glider landing blast order town chest half nest',\n", " 'ship imperial station troop droid fighter planet escape plan base fire destroy power board trench message attack port small docking bay run space compactor battle order moon wingman officer large',\n", " 'factory worker grave german camp guard ghetto train army list town stone actor polish area late successful military official run contact jewish return labor final card abuse problem end officer',\n", " 'broker stock company place office sale drug hooker floor join small month quaalude group cocaine big effect coke garage apartment business pen guy deal account party gay wedding chant pitch',\n", " 'trick machine cutter stage bird secret diary hat audience magician cage act tank trapdoor cat double box part knot version bullet show performance magic water finger collector idea left top',\n", " 'crew owner mole member identity shop informant store crime bar boy force officer academy civilian give meeting cop organization meet back envelope roof security number product protection job organized guy',\n", " 'ship light saber base imperial city asteroid cave creature droid escape planet walker small energy field ground training massive fleet powerful surface space group join shield weather vane force plan',\n", " 'guard execution mouse inmate jingle job padded cell memory chair movie prison pain place small spore watch sponge electricity mental electric infection talk show long story love die crime audience',\n", " 'patient doctor hospital island lighthouse staff warden ground building water guard mental drug orderly cliff kid dream violence interview side cell dangerous meeting vacation lot rifle live talk fire point',\n", " 'brother party business bodyguard local problem murder committee movie revenge assassination escape compound gambling pay neighborhood favor deal order return rebel show attack witness testify gaming license attempt birthday button',\n", " 'attacker order condition white chronological story black plate note number memory motel caller color reverse drug attack movie scene town clothe escort license sequence photo lie deal tattoo clue meeting',\n", " 'avatar human military tree animal ship lab banshee base grace suit ground toruk queue brother pod thanator bond fight attack form planet unobtanium warrior large soldier mission operation stop forest',\n", " 'relationship gay affair rain narrator daughter boring camcorder marijuana footage sexual dismayed image surprise idea event moment bathroom happy gunshot worry husband advertising executive materialistic ambitious realtor unsuccessful potential unhappy',\n", " 'clock future flux tower bolt lot parking capacitor street suit truck town tree fire band dance back radiation garage skateboard van square light driver plutonium amplifier minute boy road lightning',\n", " 'ship class water passenger stern necklace butler deck boat iceberg board lifeboat wreck marriage officer minute bottom sex automobile cargo hold watertight compartment crew safe drawing person sketch suicide rail',\n", " 'ship plan prison stone rocket space planet fight mom broker destroy guard daughter meet dance blow bridge security station big assistant platoon goon escape leg join reveal giant crew die',\n", " 'hotel clerk apartment school work building give hitman houseplant guy agent brother guard lobby clean store cigarette hallway drug glass grocery open moment cartoon pig cleaner member job plant form',\n", " 'white black supremacist kid brother paper jail truck boy bathroom scene window teacher office end school move suburban thing face gang group history afraid inmate wave sleep due girlfriend bedroom',\n", " 'mask government black terrorist give officer country surveillance party fear station costume stay investigation front real bed information parent soldier note die train bespectacled scarlet carson member broadcast figure system',\n", " 'drug crime rest cargo mob spider pistol prison cocaine dollar shot connection final trunk lucchese career heist mistress terminal title card bloody shut gangster scene work local associate load court',\n", " 'crew pirate island curse coin medallion chest blood immortal mysterious fall exchange long break cache rum daughter sail gold unconscious ship bay blacksmith agreement reveal loophole rescue treasure true form',\n", " 'plant ship human robot special machine pod voice garbage space platform chair escape early garage confused return repair deck plasma cannon hibernation mode hover electric kiss chute central spaceship century',\n", " 'bride fight sword face chapter orderly japanese bar blood floor yellow knife leg yakuza party member revenge red pickup flashback age early murder hospital coma mobile long restaurant list keeper',\n", " 'truck metal fire human arm grenade large floor shot foot bar elevator chip guard computer launcher cop liquid building lab pick open eye motorcycle bike helicopter form pool key shotgun',\n", " 'story ship criminal officer boat smuggler hispanic present hungarian artist office group thief cocaine station passenger figure harbor face immunity early truck part person hijacker corrupt phrase job jewel lawyer',\n", " 'clown apartment show subway murder audience joke face worker front comedy train laugh sign detective question alley condition boss hospital moment theater clip file social person full defense citizen mask',\n", " 'english army king nobleman scottish throne treachery clan garrison threat invasion battle capture magistrate mercy crowd freedom invade heir brother pilgrimage paternal land privilege grown love childhood secret soldier attempt',\n", " 'fleet imperial dark side battle shuttle team commander space station forest star sail barge moon sister generator shield fighter court final pit lightsaber rest power part attack scout face destroyer']" ] }, "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "source": [ "text_keys" ] }, { "cell_type": "code", "execution_count": null, "id": "9fc1b6c8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9d6df71f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 5 }