{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "2c2bd12b", "metadata": {}, "outputs": [], "source": [ "import re\n", "from nltk.corpus import stopwords\n", "from nltk.stem.wordnet import WordNetLemmatizer\n", "\n", "stop_words = set(stopwords.words('english'))\n", "import pandas as pd\n", "import joblib\n", "import numpy as np\n", "import nltk\n", "import spacy\n", "nlp= spacy.load('en_core_web_md')" ] }, { "cell_type": "code", "execution_count": 2, "id": "7213f06f", "metadata": {}, "outputs": [], "source": [ "texts= pd.read_csv('synopsis.tsv', sep='\\t').iloc[:,-1]" ] }, { "cell_type": "code", "execution_count": 3, "id": "ca8c0d41", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 In 1947, Andy Dufresne (Tim Robbins), a banker...\n", "1 The movie begins with a gang of men with clown...\n", "2 A young man, exhausted and delirious, washes u...\n", "3 We back out of the webbing of neurons and brai...\n", "4 Late one morning in the Hawthorne Grill, a res...\n", " ... \n", "8013 It looks like we don't have a Synopsis for thi...\n", "8014 It looks like we don't have a Synopsis for thi...\n", "8015 It looks like we don't have a Synopsis for thi...\n", "8016 It looks like we don't have a Synopsis for thi...\n", "8017 The opening scene begins with Maggie (Greta Ge...\n", "Name: 0, Length: 8018, dtype: object" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "texts" ] }, { "cell_type": "code", "execution_count": 4, "id": "43848462", "metadata": {}, "outputs": [], "source": [ "df= pd.read_csv(\"final data.tsv\", sep='\\t')" ] }, { "cell_type": "code", "execution_count": 5, "id": "5f930c0f", "metadata": {}, "outputs": [], "source": [ "df=df.sort_values(by='numVotes', ascending=False)[:len(texts)]" ] }, { "cell_type": "code", "execution_count": 6, "id": "dd94ee2e", "metadata": {}, "outputs": [], "source": [ "df['synopsis']=texts.values\n", "del texts" ] }, { "cell_type": "code", "execution_count": 7, "id": "bd336410", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titleIdtitleaverageRatingnumVotessynopsis
71144tt0111161The Shawshank Redemption9.32405502In 1947, Andy Dufresne (Tim Robbins), a banker...
162596tt0468569The Dark Knight9.02365212The movie begins with a gang of men with clown...
215067tt1375666Inception8.82122723A young man, exhausted and delirious, washes u...
83435tt0137523Fight Club8.81899050We back out of the webbing of neurons and brai...
70952tt0110912Pulp Fiction8.91868911Late one morning in the Hawthorne Grill, a res...
..................
261030tt2704380Seeing Things8.916920It looks like we don't have a Synopsis for thi...
161072tt0458364Fast Track5.516919It looks like we don't have a Synopsis for thi...
316650tt6533240Night Hunter5.916915It looks like we don't have a Synopsis for thi...
172124tt0819714The Edge of Love6.216915It looks like we don't have a Synopsis for thi...
328822tt7715202Sarkar7.016914The opening scene begins with Maggie (Greta Ge...
\n", "

8018 rows × 5 columns

\n", "
" ], "text/plain": [ " titleId title averageRating numVotes \\\n", "71144 tt0111161 The Shawshank Redemption 9.3 2405502 \n", "162596 tt0468569 The Dark Knight 9.0 2365212 \n", "215067 tt1375666 Inception 8.8 2122723 \n", "83435 tt0137523 Fight Club 8.8 1899050 \n", "70952 tt0110912 Pulp Fiction 8.9 1868911 \n", "... ... ... ... ... \n", "261030 tt2704380 Seeing Things 8.9 16920 \n", "161072 tt0458364 Fast Track 5.5 16919 \n", "316650 tt6533240 Night Hunter 5.9 16915 \n", "172124 tt0819714 The Edge of Love 6.2 16915 \n", "328822 tt7715202 Sarkar 7.0 16914 \n", "\n", " synopsis \n", "71144 In 1947, Andy Dufresne (Tim Robbins), a banker... \n", "162596 The movie begins with a gang of men with clown... \n", "215067 A young man, exhausted and delirious, washes u... \n", "83435 We back out of the webbing of neurons and brai... \n", "70952 Late one morning in the Hawthorne Grill, a res... \n", "... ... \n", "261030 It looks like we don't have a Synopsis for thi... \n", "161072 It looks like we don't have a Synopsis for thi... \n", "316650 It looks like we don't have a Synopsis for thi... \n", "172124 It looks like we don't have a Synopsis for thi... \n", "328822 The opening scene begins with Maggie (Greta Ge... \n", "\n", "[8018 rows x 5 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 8, "id": "7d9541e7", "metadata": {}, "outputs": [], "source": [ "def keep_unique(df):\n", " unique= df.synopsis.unique()\n", " obv= np.array( ['OOBBVV']*( len(df) - len(unique) ))\n", " array= np.concatenate((unique, obv))\n", " df=df[df.synopsis== array]\n", " return df" ] }, { "cell_type": "code", "execution_count": 9, "id": "32e39719", "metadata": {}, "outputs": [], "source": [ "df=keep_unique(df)" ] }, { "cell_type": "code", "execution_count": 10, "id": "4c321fdc", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titleIdtitleaverageRatingnumVotessynopsis
71144tt0111161The Shawshank Redemption9.32405502In 1947, Andy Dufresne (Tim Robbins), a banker...
162596tt0468569The Dark Knight9.02365212The movie begins with a gang of men with clown...
215067tt1375666Inception8.82122723A young man, exhausted and delirious, washes u...
83435tt0137523Fight Club8.81899050We back out of the webbing of neurons and brai...
70952tt0110912Pulp Fiction8.91868911Late one morning in the Hawthorne Grill, a res...
..................
176225tt0892769How to Train Your Dragon8.1680879A Viking boy called Hiccup (voice: Jay Baruche...
113903tt0241527Harry Potter and the Sorcerer's Stone7.6678551Lord Voldemort, an evil and powerful dark wiza...
114898tt0245429Sen to Chihiro no kamikakushi8.6674079Ten-year-old Chihiro (voice: Daveigh Chase in ...
185800tt1045658Silver Linings Playbook7.7672181Pat Solitano (Bradley Cooper), who has bipolar...
238101tt1877832X-Men: Days of Future Past7.9670447Patrick Stewart's Charles Xavier recounts how ...
\n", "

132 rows × 5 columns

\n", "
" ], "text/plain": [ " titleId title averageRating \\\n", "71144 tt0111161 The Shawshank Redemption 9.3 \n", "162596 tt0468569 The Dark Knight 9.0 \n", "215067 tt1375666 Inception 8.8 \n", "83435 tt0137523 Fight Club 8.8 \n", "70952 tt0110912 Pulp Fiction 8.9 \n", "... ... ... ... \n", "176225 tt0892769 How to Train Your Dragon 8.1 \n", "113903 tt0241527 Harry Potter and the Sorcerer's Stone 7.6 \n", "114898 tt0245429 Sen to Chihiro no kamikakushi 8.6 \n", "185800 tt1045658 Silver Linings Playbook 7.7 \n", "238101 tt1877832 X-Men: Days of Future Past 7.9 \n", "\n", " numVotes synopsis \n", "71144 2405502 In 1947, Andy Dufresne (Tim Robbins), a banker... \n", "162596 2365212 The movie begins with a gang of men with clown... \n", "215067 2122723 A young man, exhausted and delirious, washes u... \n", "83435 1899050 We back out of the webbing of neurons and brai... \n", "70952 1868911 Late one morning in the Hawthorne Grill, a res... \n", "... ... ... \n", "176225 680879 A Viking boy called Hiccup (voice: Jay Baruche... \n", "113903 678551 Lord Voldemort, an evil and powerful dark wiza... \n", "114898 674079 Ten-year-old Chihiro (voice: Daveigh Chase in ... \n", "185800 672181 Pat Solitano (Bradley Cooper), who has bipolar... \n", "238101 670447 Patrick Stewart's Charles Xavier recounts how ... \n", "\n", "[132 rows x 5 columns]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": null, "id": "514572db", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 11, "id": "1bcdbebd", "metadata": {}, "outputs": [], "source": [ "def pre_process(text, allowed_pos=['NOUN',\"ADJ\",'JJ','JJR', 'VB']):\n", " doc= nlp(text)\n", " new_text=[ token.lemma_ for token in doc if token.pos_ in allowed_pos and token.tag_!= 'NNP']\n", " final=' '.join(new_text)\n", "\n", " return final" ] }, { "cell_type": "code", "execution_count": 12, "id": "b57e6e2f", "metadata": {}, "outputs": [], "source": [ "docs = df['synopsis'].apply(lambda x:pre_process(x)).to_list()" ] }, { "cell_type": "code", "execution_count": null, "id": "a2513c64", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 25, "id": "dba708de", "metadata": {}, "outputs": [], "source": [ "from sklearn.feature_extraction.text import CountVectorizer\n", "#docs = docs.tolist()\n", "#create a vocabulary of words, \n", "cv=CountVectorizer(max_df=0.95, # ignore words that appear in 95% of documents\n", " max_features=100000, # the size of the vocabulary\n", " ngram_range=(1,3), # vocabulary contains single words, bigrams, trigrams\n", " lowercase=False\n", " )\n", "word_count_vector=cv.fit_transform(docs)" ] }, { "cell_type": "code", "execution_count": 26, "id": "7b9c5196", "metadata": {}, "outputs": [], "source": [ "def top_n_words(corpus, n=50):\n", " vec = CountVectorizer().fit(corpus)\n", " bag_of_words = vec.transform(corpus)\n", " sum_words = bag_of_words.sum(axis=0) \n", " words_freq = sorted([(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()], key = lambda x: x[1], reverse=True)\n", " words=[w[0] for w in words_freq[:n] ]\n", " return words" ] }, { "cell_type": "code", "execution_count": 27, "id": "50f775e8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TfidfTransformer()" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.feature_extraction.text import TfidfTransformer\n", "\n", "tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True)\n", "tfidf_transformer.fit(word_count_vector)" ] }, { "cell_type": "code", "execution_count": 28, "id": "e83ed404", "metadata": {}, "outputs": [], "source": [ "def sort_coo(coo_matrix):\n", " tuples = zip(coo_matrix.col, coo_matrix.data)\n", " return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)" ] }, { "cell_type": "code", "execution_count": 29, "id": "aa859d50", "metadata": {}, "outputs": [], "source": [ "def extract_topn_from_vector(feature_names, sorted_items, topn=10):\n", " \"\"\"get the feature names and tf-idf score of top n items\"\"\"\n", " \n", " #use only topn items from vector\n", " sorted_items = sorted_items[:topn]\n", "\n", " score_vals = []\n", " feature_vals = []\n", "\n", " for idx, score in sorted_items:\n", " fname = feature_names[idx]\n", " \n", " #keep track of feature name and its corresponding score\n", " score_vals.append(round(score, 3))\n", " feature_vals.append(feature_names[idx])\n", "\n", " #create a tuples of feature,score\n", " #results = zip(feature_vals,score_vals)\n", " results= {}\n", " for idx in range(len(feature_vals)):\n", " results[feature_vals[idx]]=score_vals[idx]\n", " \n", " return results\n", "\n", "# get feature names\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 30, "id": "bf66b02a", "metadata": {}, "outputs": [], "source": [ "feature_names=cv.get_feature_names()\n", "freq_words=top_n_words(docs)" ] }, { "cell_type": "code", "execution_count": 31, "id": "e77f7b36", "metadata": {}, "outputs": [], "source": [ "def get_keywords(idx, docs, n=10):\n", "\n", " #generate tf-idf for the given document\n", " #print(docs[idx])\n", " tf_idf_vector=tfidf_transformer.transform(cv.transform([docs[idx]]))\n", "\n", " #sort the tf-idf vectors by descending order of scores\n", " sorted_items=sort_coo(tf_idf_vector.tocoo())\n", "\n", " #extract only the top n; n here is 10\n", " keywords=extract_topn_from_vector(feature_names,sorted_items,2*n) #returns a dict\n", " keywords=list(keywords.keys()) #dict.keys to list\n", " keywords= [k for k in keywords if k not in freq_words][:n] # not in top n words\n", " return keywords\n", "\n" ] }, { "cell_type": "code", "execution_count": 32, "id": "808cc336", "metadata": {}, "outputs": [], "source": [ "def print_results(idx,keywords, df):\n", " # now print the results\n", " print(\"\\n=====Title=====\")\n", " print(df.iloc[idx,1])\n", " print(\"\\n===Keywords===\")\n", " #for k in keywords:\n", " print(keywords)\n" ] }, { "cell_type": "code", "execution_count": 33, "id": "9b9f477c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=====Title=====\n", "The Dark Knight\n", "\n", "===Keywords===\n", "['mob', 'coin', 'mob member', 'clown', 'ferry', 'member', 'cell phone', 'mayor', 'bank', 'cell', 'goon', 'boat', 'phone', 'detonator', 'cop', 'sonar', 'clean side', 'mobster', 'criminal', 'side', 'dirty', 'location', 'convict', 'clean', 'chaos', 'card', 'city', 'mob money', 'honor guard', 'half face']\n" ] } ], "source": [ "idx=1\n", "keywords=get_keywords(idx, docs,30)\n", "print_results(idx,keywords, df)" ] }, { "cell_type": "code", "execution_count": null, "id": "7ad89c83", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 34, "id": "672de4dc", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2225" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(pd.read_csv('https://raw.githubusercontent.com/jangedoo/jange/master/dataset/bbc.csv'))" ] }, { "cell_type": "code", "execution_count": 35, "id": "b65e04c0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'day', 'tommorow', 'mailbox', 'opened'}\n", "{'yesterday', 'mailbox', 'day', 'tommorow', 'opened'}\n", "similarity: 0.5773502691896258\n" ] } ], "source": [ "from nltk.corpus import stopwords\n", "from nltk.tokenize import word_tokenize\n", "X =\"i opened mailbox day before tommorow\"\n", "Y =\"yesterday mailbox was opened by me\"\n", " \n", "# tokenization\n", "X_list = word_tokenize(X) \n", "Y_list = word_tokenize(Y)\n", " \n", "# sw contains the list of stopwords\n", "sw = stopwords.words('english') \n", "l1 =[];l2 =[]\n", " \n", "# remove stop words from the string\n", "X_set = {w for w in X_list if not w in sw} \n", "Y_set = {w for w in Y_list if not w in sw}\n", "print(X_set)\n", "# form a set containing keywords of both strings \n", "rvector = X_set.union(Y_set) \n", "print(rvector)\n", "for w in rvector:\n", " if w in X_set: l1.append(1) # create a vector\n", " else: l1.append(0)\n", " if w in Y_set: l2.append(1)\n", " else: l2.append(0)\n", "c = 0\n", " \n", "# cosine formula \n", "for i in range(len(rvector)):\n", " c+= l1[i]*l2[i]\n", "cosine = c / float((sum(l1)*sum(l2))**0.5)\n", "print(\"similarity: \", cosine)" ] }, { "cell_type": "code", "execution_count": 37, "id": "990e6421", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "NN\n", "NNP\n", "VBZ\n" ] } ], "source": [ "t=nlp('batman Joker kills')\n", "for i in t:\n", " print(i.tag_)" ] }, { "cell_type": "code", "execution_count": null, "id": "bad5bf21", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "149103de", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 5 }