1  from brownclustering.helpers import *
   2  from brownclustering.core import *
   3  
   4  
   5  class BrownClusteringValidation(BrownClustering):
   6      def __init__(self, corpus, m):
   7          super().__init__(corpus, m)
   8  
   9          self.helpers = []
  10          self.helpers.append(RawClusteringHelper(corpus))
  11          self.helpers.append(ModerateClusteringHelper(corpus))
  12          self.helpers.append(EnhancedClusteringHelper(corpus))
  13  
  14      @staticmethod
  15      def arg_max(_benefit, _helper):
  16          max_benefit = float('-inf')
  17          best_merge = None
  18          for i in range(_benefit.shape[0]):
  19              for j in range(i + 1, _benefit.shape[1]):
  20                  if max_benefit < _benefit[i, j]:
  21                      max_benefit = _benefit[i, j]
  22                      best_merge = (i, j)
  23          return best_merge
  24  
  25      def validate(self):
  26          words = self.ranks(self.vocabulary)
  27          tops = words[0:self.m]
  28  
  29          count = 0
  30          _words = []
  31          for w in tops:
  32              _words.append(w[0])
  33              if len(_words) > 2:
  34                  for helper in self.helpers:
  35                      helper.append_cluster(_words)
  36                  _words = []
  37                  count += 1
  38  
  39          if len(_words) > 0:
  40              for helper in self.helpers:
  41                  helper.append_cluster(_words)
  42              count += 1
  43  
  44          itr = 0
  45          for w in words[self.m:]:
  46              itr += 1
  47              print(str(itr) + "\t" + str(datetime.datetime.now()))
  48  
  49              last_benefit = None
  50              for helper in self.helpers:
  51                  helper.append_cluster([w[0]])
  52                  _benefit = helper.compute_benefit()
  53                  if last_benefit is not None:
  54                      print(_benefit - last_benefit)
  55                  last_benefit = _benefit
  56                  best_merge = self.arg_max(_benefit, helper)
  57  
  58              print(best_merge)
  59              for helper in self.helpers:
  60                  helper.merge_clusters(best_merge[0], best_merge[1])
  61  
  62          for _ in range(count - 1):
  63              itr += 1
  64              print(str(itr) + "\t" + str(datetime.datetime.now()))
  65              last_benefit = None
  66              for helper in self.helpers:
  67                  _benefit = helper.compute_benefit()
  68                  if last_benefit is not None:
  69                      print(_benefit - last_benefit)
  70                  best_merge = self.arg_max(_benefit, helper)
  71  
  72              print(best_merge)
  73              for helper in self.helpers:
  74                  helper.merge_clusters(best_merge[0], best_merge[1])