{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "954e9fd9", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import nltk\n", "from nltk.corpus import stopwords\n", "stopwords= stopwords.words('english')\n", "stopwords.extend([',','`','?',\"'s\", \"''\", '``',\"n't\",\"(\", \")\", ';',\"'\",'re','!','--','-','*',\"'re\",':'])" ] }, { "cell_type": "code", "execution_count": 2, "id": "5c9f32c2", "metadata": {}, "outputs": [], "source": [ "stopwords.extend([i.capitalize() for i in stopwords])" ] }, { "cell_type": "code", "execution_count": 3, "id": "18a56afa", "metadata": {}, "outputs": [], "source": [ "s1= pd.read_csv('data/unique synopsis.tsv', sep='\\t')\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "f07f0c11", "metadata": { "scrolled": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0
0In 1947, Andy Dufresne (Tim Robbins), a banker...
1The movie begins with a gang of men with clown...
2A young man, exhausted and delirious, washes u...
3We back out of the webbing of neurons and brai...
4Late one morning in the Hawthorne Grill, a res...
\n", "
" ], "text/plain": [ " 0\n", "0 In 1947, Andy Dufresne (Tim Robbins), a banker...\n", "1 The movie begins with a gang of men with clown...\n", "2 A young man, exhausted and delirious, washes u...\n", "3 We back out of the webbing of neurons and brai...\n", "4 Late one morning in the Hawthorne Grill, a res..." ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s1.head()" ] }, { "cell_type": "code", "execution_count": 5, "id": "a52e3b31", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "9643" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(s1)" ] }, { "cell_type": "code", "execution_count": 6, "id": "74d6ffd7", "metadata": {}, "outputs": [], "source": [ "from nltk.tokenize import word_tokenize" ] }, { "cell_type": "code", "execution_count": 7, "id": "50095664", "metadata": {}, "outputs": [], "source": [ "texts= s1.dropna(axis=0).values.reshape(-1,).astype('object')" ] }, { "cell_type": "code", "execution_count": 8, "id": "d1ea9fdd", "metadata": {}, "outputs": [], "source": [ "texts=' '.join(texts)" ] }, { "cell_type": "code", "execution_count": 9, "id": "10634606", "metadata": {}, "outputs": [], "source": [ "texts=' '.join(texts.split('.')).lower()" ] }, { "cell_type": "code", "execution_count": null, "id": "3c3adcaf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 10, "id": "cd909ad7", "metadata": {}, "outputs": [], "source": [ "words= np.array(word_tokenize(texts)) ### chewck" ] }, { "cell_type": "code", "execution_count": 11, "id": "661ed7bd", "metadata": {}, "outputs": [], "source": [ "words=np.array([w for w in words if not w in stopwords])" ] }, { "cell_type": "code", "execution_count": 12, "id": "2fe22ff2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "5641411" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(words)" ] }, { "cell_type": "code", "execution_count": 13, "id": "62986f26", "metadata": {}, "outputs": [], "source": [ "features=np.array(list(set(words)))" ] }, { "cell_type": "code", "execution_count": 14, "id": "0373e926", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "128962" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(features)" ] }, { "cell_type": "code", "execution_count": null, "id": "c0f901b4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 15, "id": "ddc5ac8d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([], dtype=int64)" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.where(words== 'in')[0]" ] }, { "cell_type": "code", "execution_count": 16, "id": "35406a83", "metadata": {}, "outputs": [], "source": [ "def count_f(array):\n", " unique, counts = np.unique(array, return_counts=True)\n", " if len(counts)>1:\n", " counts= counts/len(counts)\n", " else:\n", " counts= counts/counts\n", " array=dict(zip(unique, counts))\n", " array={k: v for k, v in sorted(array.items(), key=lambda item: item[1], reverse=True)} #set to true\n", " #print(array)\n", " return array" ] }, { "cell_type": "code", "execution_count": 17, "id": "d7a75ae7", "metadata": {}, "outputs": [], "source": [ "def next_previous(feature):\n", " indexs= np.where(words== feature)[0]\n", " \n", " next_word= count_f(words[indexs+1])\n", " previous_word= count_f(words[indexs-1])\n", "\n", " return np.vstack((previous_word, next_word)).astype('object').T" ] }, { "cell_type": "code", "execution_count": null, "id": "38253854", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "977a0bc0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3acfa8e6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 18, "id": "7bdf69e7", "metadata": {}, "outputs": [], "source": [ "def word_cha(features):\n", " final={}\n", " for feature in features:\n", " final[feature]= next_previous(feature)\n", " return final" ] }, { "cell_type": "code", "execution_count": 27, "id": "db74d8c2", "metadata": {}, "outputs": [], "source": [ "def find_most_probable(word,num=10):\n", " try:\n", " words=next_previous(word)\n", " previous_words= np.array(list(words[0][0].keys())).astype('object')[:num]\n", " next_words= np.array(list(words[0][1].keys())).astype('object')[:num]\n", " previous_words_prob= np.array(list(words[0][0].values()))\n", " prob_words=np.array([])\n", " for i,previous_word in enumerate(previous_words):\n", " #print(i, 'p',previous_word)\n", " #c=previous_words_prob[i]\n", " #print(f'c= {c}')\n", " n=next_previous(previous_word)\n", " #print(n)\n", "\n", " next_words_child= np.array(list(n[0][1].keys())).astype('object')\n", " prob_words= np.append(prob_words, next_words_child)\n", "\n", " if i