1  {
   2   "cells": [
   3    {
   4     "cell_type": "code",
   5     "execution_count": 1,
   6     "id": "59b61714",
   7     "metadata": {},
   8     "outputs": [],
   9     "source": [
  10      "import pandas as pd\n",
  11      "import numpy as np\n",
  12      "from nltk.corpus import stopwords\n",
  13      "stopwords= stopwords.words('english')\n",
  14      "stopwords.extend([',','`','?',\"'s\", \"''\", '``',\"n't\",\"(\", \")\", ';',\"'\",'re','!','--','-','*',\"'re\"])"
  15     ]
  16    },
  17    {
  18     "cell_type": "code",
  19     "execution_count": 2,
  20     "id": "05b4de3b",
  21     "metadata": {},
  22     "outputs": [],
  23     "source": [
  24      "import unittest\n",
  25      "from brownclustering import Corpus\n",
  26      "from brownclustering import BrownClustering\n",
  27      "from brownclustering.validation import BrownClusteringValidation"
  28     ]
  29    },
  30    {
  31     "cell_type": "code",
  32     "execution_count": 3,
  33     "id": "e0861ca3",
  34     "metadata": {},
  35     "outputs": [],
  36     "source": [
  37      "texts= pd.read_csv('Final2.0.tsv', sep='\\t').dropna(axis=0).iloc[:,-2].values.reshape(-1,).astype('object')"
  38     ]
  39    },
  40    {
  41     "cell_type": "code",
  42     "execution_count": 4,
  43     "id": "f1d3aa7e",
  44     "metadata": {},
  45     "outputs": [
  46      {
  47       "data": {
  48        "text/plain": [
  49         "'prison,warden,inmate,red remark,solitary,rock hammer,parole,cell,rock,library,prisoner,hammer,guard,message wooden beam,message wooden,state,prison library,giant poster,wooden beam,wall,poster,red,remark,diploma,book,rest life,life prison,job,prison guard,infirmary,school diploma,prison library state,library state,inmate fat ass,inmate fat,high school diploma,commemorate year,state senate,fat ass,financial,record,hope,banker,tax return,police squad car,cold beer,shoe,stone,warden office,project,night'"
  50        ]
  51       },
  52       "execution_count": 4,
  53       "metadata": {},
  54       "output_type": "execute_result"
  55      }
  56     ],
  57     "source": [
  58      "texts[0]"
  59     ]
  60    },
  61    {
  62     "cell_type": "code",
  63     "execution_count": null,
  64     "id": "dae810d8",
  65     "metadata": {},
  66     "outputs": [],
  67     "source": []
  68    },
  69    {
  70     "cell_type": "code",
  71     "execution_count": 5,
  72     "id": "12644df4",
  73     "metadata": {},
  74     "outputs": [],
  75     "source": [
  76      "corpus=[]\n",
  77      "for text in texts:\n",
  78      "    text=text.split(',')\n",
  79      "    corpus.append(text)"
  80     ]
  81    },
  82    {
  83     "cell_type": "code",
  84     "execution_count": null,
  85     "id": "ea3fd0a0",
  86     "metadata": {},
  87     "outputs": [],
  88     "source": [
  89      "corpus=Corpus(corpus)"
  90     ]
  91    },
  92    {
  93     "cell_type": "code",
  94     "execution_count": null,
  95     "id": "ef911661",
  96     "metadata": {},
  97     "outputs": [],
  98     "source": []
  99    }
 100   ],
 101   "metadata": {
 102    "kernelspec": {
 103     "display_name": "Python 3",
 104     "language": "python",
 105     "name": "python3"
 106    },
 107    "language_info": {
 108     "codemirror_mode": {
 109      "name": "ipython",
 110      "version": 3
 111     },
 112     "file_extension": ".py",
 113     "mimetype": "text/x-python",
 114     "name": "python",
 115     "nbconvert_exporter": "python",
 116     "pygments_lexer": "ipython3",
 117     "version": "3.8.8"
 118    }
 119   },
 120   "nbformat": 4,
 121   "nbformat_minor": 5
 122  }
 123