← [BACK TO REPOSITORY]    Media: helpers.cpython-38.pyc (13.2 KB)
Format: application/x-python-code
⬇ Download File (13.2 KB) 
   1  U
   2  
   3  .R�`35�@sRddlZddlZGdd�d�ZGdd�de�ZGdd�de�ZGdd	�d	e�ZdS)
   4  �Nc@sDeZdZdd�Zdd�Zdd�Zdd�Zd	d
   5  �Zdd�Zd
   6  d�Z	dS)�ClusteringHelpercCs|j|_|j|_|j|_dS�N)�n�unigrams�bigrams��self�corpus�r
   7  �9C:\Users\PC\Pytho\Projects\NLP\brownclustering\helpers.py�__init__szClusteringHelper.__init__cCs2d}|D]$}|D]}||j�||fd�7}qq|S�Nr)r�get)rZcluster1Zcluster2�_count�w1�w2r
   8  r
   9  r�
  10  count_bigramss
  11  zClusteringHelper.count_bigramscCs
  12  t��dSr��NotImplementedError�r�wordsr
  13  r
  14  r�append_clusterszClusteringHelper.append_clustercCs
  15  t��dSrr�r�i�jr
  16  r
  17  r�merge_clustersszClusteringHelper.merge_clusterscCs
  18  t��dSrr�rr
  19  r
  20  r�get_clustersszClusteringHelper.get_clusterscCs
  21  t��dSrr�rrr
  22  r
  23  r�get_clusterszClusteringHelper.get_clustercCs
  24  t��dSrrrr
  25  r
  26  r�compute_benefitsz ClusteringHelper.compute_benefitN)
  27  �__name__�
  28  __module__�__qualname__rrrrrrr r
  29  r
  30  r
  31  rrsrcsteZdZ�fdd�Zdd�Zdd�Zdd�Zd	d
  32  �Zdd�Zd
  33  d�Z	dd�Z
  34  dd�Zdd�Zdd�Z
  35  dd�Z�ZS)�EnhancedClusteringHelpercs\t��|�d|_g|_tjdtd�|_tjdtd�|_tjdtd�|_	tjdtd�|_
  36  dS�Nr��dtype)rr)�superr�m�clusters�np�zeros�float�p1�p2�q2�l2r��	__class__r
  37  rr$sz!EnhancedClusteringHelper.__init__cCs�tj|j|jddd�|_d}|D]}||j|7}q ||j|j|j<tj|j|jddd�|_tj|j|jddd�|_t|j�D]L}|�||j	|�|j|j|j|f<|�|j	||�|j|j||jf<q�|�||�|j|j|j|jf<tj|j
  38  |jddd�|_
  39  tj|j
  40  |jddd�|_
  41  t|j�D]:}|�|j|�|j
  42  |j|f<|�||j�|j
  43  ||jf<�q(|�|j|j�|j
  44  |j|jf<tj|j|jddd�|_tj|j|jddd�|_|jd|_|j	�
  45  |���|��dS�z
  46          O(m (+ n))
  47          r��axis�N)r+�insertr.r)rrr/�rangerr*r0�_qr1�append�copy�_update_deltas�rr�_sum�wrr
  48  r
  49  rr-s,$& z'EnhancedClusteringHelper.append_clustercCs:t|jd�D�]$}|�||jd�|j||jdf<t|d|jd�D]�}|j||f|j||jdf8<|j||f|j||jdf8<|j||f|j|jd|f8<|j||f|j|jd|f8<|j||f|�|||jd�7<|j||f|�|jd||�7<qJqdS)Nr7)r9r)�_deltar1r0�_q_l�_q_rrr
  50  r
  51  rr=Ns"&&&&&z'EnhancedClusteringHelper._update_deltascCs
  52  |j��Sr�r*r<rr
  53  r
  54  rrZsz%EnhancedClusteringHelper.get_clusterscCs|j|��SrrDrr
  55  r
  56  rr]sz$EnhancedClusteringHelper.get_clustercCs�t|j�D�]}t|d|j�D]�}d}||�|||�7}||�|||�7}||�|||�7}||�|||�7}||j||f8}||j||f8}||j||f8}||j||f8}||j||f8}||j||f8}||j||f8}||j||f8}|j||f|8<q q
  57  |j|�|j|�|j|=|jd|_|j||j||j|<t	j
  58  |j|dd�|_|j|dd�f|j|dd�f|j|dd�f<|jdd�|f|jdd�|f|jdd�|f<t	j
  59  |j|dd�|_t	j
  60  |j|dd�|_t	j
  61  |j|dd�|_t	j
  62  |j|dd�|_t|j�D]2}|�||�|j||f<|�||�|j||f<�q0|�||�|j||f<t	j
  63  |j|dd�|_t	j
  64  |j|dd�|_t|j�D]�}t|d|j�D]�}d}||�|||�7}||�|||�7}||j||f8}||j||f8}||j||f8}||j||f8}|j||f|7<�q��q�t|�D]}|�
  65  ||�|j||f<�qZt|d|j�D]}|�
  66  ||�|j||f<�q�dS)Nr7rr5)r9r)rBrCr0r1r*�extendr.r+�deleter/r:rA)rrr�_i�_j�_tmp�_x�xr
  67  r
  68  rr`s^22z'EnhancedClusteringHelper.merge_clusterscCs
  69  |j��Sr)r1r<rr
  70  r
  71  rr �sz(EnhancedClusteringHelper.compute_benefitcCsP|j||f|j||f}|j||j|}|j|}|t�|||�S�z
  72          O(1)
  73          �r/r.�math�log�rrGrHrJ�pcx�pc�pxr
  74  r
  75  rrB�s
  76  zEnhancedClusteringHelper._q_lcCsP|j||f|j||f}|j||j|}|j|}|t�|||�SrLrM�rrJrGrH�pxcrRrSr
  77  r
  78  rrC�s
  79  zEnhancedClusteringHelper._q_rcCsv|j||f|j||f|j||f|j||f}|j||j|}|j||j|}|t�|||�SrLrM�rrGrHrUrRrSr
  80  r
  81  r�_q_x�s8zEnhancedClusteringHelper._q_xcCs8|j||f}|j|}|j|}|t�|||�SrLrM�rrGrJrQrRrSr
  82  r
  83  rr:�s
  84  
  85  zEnhancedClusteringHelper._qcsT�j��j���j�dd�f�j�dd�f��jdd��f�jdd��f�����fdd�}����fdd�}d}t�j�D]j}|�j�|f8}|�j|�f8}|�j�|f8}|�j|�f8}|�ks�|�kr�q�|||�7}q�||�d7}|�j��f7}|�j��f7}|�j��f7}|�j��f7}|S)NcsJ�|}�|}�}�j|}|t�|||�|t�|||�Sr)r.rNrO�rJ�pij�pji�pi�pj��
  86  count_2_new_e�
  87  count_2_new_s�count_i_newrr
  88  r�
  89  _weight_new_1�s
  90  
  91  z6EnhancedClusteringHelper._delta.<locals>._weight_new_1csp�j��f�j��f�j��f�j��f}|}�}�}|t�|||�|t�|||�Sr)r/rNrO�rZr[r\r]�rarrrr
  92  r�
  93  _weight_new_2�s
  94  8z6EnhancedClusteringHelper._delta.<locals>._weight_new_2r�)r.r/r9r)r0�rrrrbre�lossrKr
  95  �r_r`rarrrrrA�s($$zEnhancedClusteringHelper._delta)r!r"r#rrr=rrrr rBrCrWr:rA�
  96  __classcell__r
  97  r
  98  r2rr$#s	!@
  99  		
 100  r$csVeZdZ�fdd�Zdd�Zdd�Zdd�Zd	d
 101  �Zddd
 102  �Zdd�Z	dd�Z
 103  �ZS)�RawClusteringHelpercst��|�g|_dSr)r(rr*rr2r
 104  rr�szRawClusteringHelper.__init__cCs|j�|���dSr)r*r;r<rr
 105  r
 106  rr�sz"RawClusteringHelper.append_clustercCs"|j|�|j|�|j|=dSr)r*rErr
 107  r
 108  rr�sz"RawClusteringHelper.merge_clusterscCs
 109  |j��SrrDrr
 110  r
 111  rr�sz RawClusteringHelper.get_clusterscCs|j|��SrrDrr
 112  r
 113  rrszRawClusteringHelper.get_clusterNcCsPd}|dkr&|D]}||j|7}qn&|D] }|D]}||j||f7}q2q*|Sr
 114  )rr)r�c1�c2�retr@rrr
 115  r
 116  r�countszRawClusteringHelper.countc
 117  s�t|�}t��t|�D]}��||��|<qt|�D].}t|�D] }��||||��||f<qBq6d��fdd�	}d}t|�D]@}t|�D]2}||||�t�|||�||�||��7}q�q�|S)Ncs(|dkr�|�jS�||f�jSr)r)rGrH��countsrr
 118  r�_probszERawClusteringHelper.compute_average_mutual_information.<locals>._probr)N)�len�dictr9rorNrO)rr*�krrrrrnr
 119  rpr�"compute_average_mutual_informations 2z6RawClusteringHelper.compute_average_mutual_informationcCs�|�|j�}t|j�}t�||f�}t|�D]V}t|d|�D]B}dd�|jD�}||�||�||=|�|�||||f<q>q,|S)Nr7cSsg|]}|���qSr
 120  )r<)�.0rKr
 121  r
 122  r�
 123  <listcomp>+sz7RawClusteringHelper.compute_benefit.<locals>.<listcomp>)rvr*rsr+r,r9rE)r�tmpru�_benefitrrZclusters_candidater
 124  r
 125  rr "s
 126  z#RawClusteringHelper.compute_benefit)N)r!r"r#rrrrrrorvr rjr
 127  r
 128  r2rrk�s
 129  rkcsleZdZ�fdd�Zdd�Zdd�Zdd�Zd	d
 130  �Zdd�Zd
 131  d�Z	dd�Z
 132  dd�Zdd�Zdd�Z
 133  �ZS)�ModerateClusteringHelpercs<t��|�d|_g|_tjdtd�|_tjdtd�|_dSr%)	r(rr)r*r+r,r-�counts_1�counts_2rr2r
 134  rr3s
 135  z!ModerateClusteringHelper.__init__cCs�tj|j|jddd�|_d}|D]}||j|7}q ||j|j<tj|j|jddd�|_tj|j|jddd�|_t|j�D]@}|�||j|�|j|j|f<|�|j||�|j||jf<qz|�||�|j|j|jf<|jd|_|j�	|�
 136  ��dSr4)r+r8r|r)rr}r9rr*r;r<r>r
 137  r
 138  rr:s z'ModerateClusteringHelper.append_clustercCs
 139  |j��SrrDrr
 140  r
 141  rrOsz%ModerateClusteringHelper.get_clusterscCs|j|��SrrDrr
 142  r
 143  rrRsz$ModerateClusteringHelper.get_clustercCs�|j||j||j|<tj|j|dd�|_|j|dd�f|j|dd�f|j|dd�f<|jdd�|f|jdd�|f|jdd�|f<tj|j|dd�|_tj|j|dd�|_|j|�|j|�|j|=|jd|_dS)Nrr5r7)r|r+rFr}r*rEr)rr
 144  r
 145  rrUs22z'ModerateClusteringHelper.merge_clusterscCsTtj|j|jftd�}t|j�D].}t|d|j�D]}|�||�|||f<q4q |S)Nr&r7)r+r,r)r-r9rA)r�deltasrrr
 146  r
 147  rr bs
 148  z(ModerateClusteringHelper.compute_benefitcCsb|j||f|j||f|j}|j||j||j}|j||j}|t�|||�SrL�r}rr|rNrOrPr
 149  r
 150  rrBis"zModerateClusteringHelper._q_lcCsb|j||f|j||f|j}|j||j||j}|j||j}|t�|||�SrLrrTr
 151  r
 152  rrCss"zModerateClusteringHelper._q_rcCs�|j||f|j||f|j||f|j||f|j}|j||j||j}|j||j||j}|t�|||�SrLrrVr
 153  r
 154  rrW|s>zModerateClusteringHelper._q_xcCsJ|j||f|j}|j||j}|j||j}|t�|||�SrLrrXr
 155  r
 156  rr:�szModerateClusteringHelper._qcsD�j��j���j�dd�f�j�dd�f��jdd��f�jdd��f�����fdd�}����fdd�}d}t�j�D]b}|���|�8}|��|��8}|���|�8}|��|��8}|�ks�|�kr�q�|||�7}q�||�d7}|�����7}|�����7}|�����7}|�����7}|S)z
 157          O(m)
 158          Ncsb�|�j}�|�j}��j}�j|�j}|t�|||�|t�|||�Sr)rr|rNrOrYr^r
 159  rrb�s
 160  
 161  z6ModerateClusteringHelper._delta.<locals>._weight_new_1cs��j��f�j��f�j��f�j��f�j}|}��j}��j}|t�|||�|t�|||�Sr)r}rrNrOrcrdr
 162  rre�s
 163  >
 164  
 165  z6ModerateClusteringHelper._delta.<locals>._weight_new_2rrf)r|r}r9r)r:rgr
 166  rirrA�s($$zModerateClusteringHelper._delta)r!r"r#rrrrrr rBrCrWr:rArjr
 167  r
 168  r2rr{2s
 169  
 170  		
 171  r{)rN�numpyr+rr$rkr{r
 172  r
 173  r
 174  r�<module>sOA