taskswithcode/semantic_clustering

1from scipy.spatial.distance import cosine2import argparse3import json4import pdb5import torch6import torch.nn.functional as F7import numpy as np8import time9from collections import OrderedDict10 11 12class TWCClustering:13    def __init__(self):14        print("In Zscore  Clustering")15 16    def compute_matrix(self,embeddings):17        #print("Computing similarity matrix ...)")18        embeddings= np.array(embeddings)19        start = time.time()20        vec_a = embeddings.T #vec_a shape (1024,)21        vec_a = vec_a/np.linalg.norm(vec_a,axis=0) #Norm is along axis 0 - rows22        vec_a = vec_a.T #vec_a shape becomes (,1024)23        similarity_matrix = np.inner(vec_a,vec_a)24        end = time.time()25        time_val = (end-start)*100026        #print(f"Similarity matrix computation complete. Time taken:{(time_val/(1000*60)):.2f}  minutes")27        return similarity_matrix28        29    def get_terms_above_threshold(self,matrix,embeddings,pivot_index,threshold):30        run_index = pivot_index31        picked_arr = []32        while (run_index < len(embeddings)):33            if (matrix[pivot_index][run_index] >= threshold):34                picked_arr.append(run_index)35            run_index += 136        return picked_arr37 38    def update_picked_dict_arr(self,picked_dict,arr):39        for i in range(len(arr)):40            picked_dict[arr[i]] = 141 42    def update_picked_dict(self,picked_dict,in_dict):43        for key in in_dict:44            picked_dict[key] = 145 46    def find_pivot_subgraph(self,pivot_index,arr,matrix,threshold,strict_cluster = True):47        center_index = pivot_index48        center_score = 049        center_dict = {}50        for i in range(len(arr)):51            node_i_index = arr[i]52            running_score = 053            temp_dict = {}54            for j in range(len(arr)):55                node_j_index = arr[j]56                cosine_dist = matrix[node_i_index][node_j_index]57                if ((cosine_dist < threshold) and strict_cluster):58                    continue59                running_score += cosine_dist60                temp_dict[node_j_index] = cosine_dist61            if (running_score > center_score):62                center_index = node_i_index63                center_dict = temp_dict64                center_score = running_score65        sorted_d = OrderedDict(sorted(center_dict.items(), key=lambda kv: kv[1], reverse=True))66        return  {"pivot_index":center_index,"orig_index":pivot_index,"neighs":sorted_d}67         68 69    def update_overlap_stats(self,overlap_dict,cluster_info):70        arr = list(cluster_info["neighs"].keys())71        for val in arr:72            if (val not in overlap_dict):73                overlap_dict[val] = 174            else:75                overlap_dict[val] += 176 77    def bucket_overlap(self,overlap_dict):78        bucket_dict = {}79        for key in overlap_dict:80            if (overlap_dict[key] not in bucket_dict):81                bucket_dict[overlap_dict[key]] = 182            else:83                bucket_dict[overlap_dict[key]] += 184        sorted_d = OrderedDict(sorted(bucket_dict.items(), key=lambda kv: kv[1], reverse=False))85        return sorted_d86 87    def merge_clusters(self,ref_cluster,curr_cluster):88        dup_arr = ref_cluster.copy()89        for j in range(len(curr_cluster)):90            if (curr_cluster[j] not in dup_arr):91                ref_cluster.append(curr_cluster[j]) 92                93 94    def non_overlapped_clustering(self,matrix,embeddings,threshold,mean,std,cluster_dict):95        picked_dict = {}96        overlap_dict = {}97        candidates = []98    99        for i in range(len(embeddings)):100            if (i in picked_dict):101                continue102            zscore = mean + threshold*std103            arr = self.get_terms_above_threshold(matrix,embeddings,i,zscore)104            candidates.append(arr)105            self.update_picked_dict_arr(picked_dict,arr)106    107        # Merge arrays to create non-overlapping sets108        run_index_i = 0109        while (run_index_i < len(candidates)):110            ref_cluster = candidates[run_index_i]111            run_index_j = run_index_i + 1112            found = False113            while (run_index_j < len(candidates)): 114                curr_cluster = candidates[run_index_j]115                for k in range(len(curr_cluster)):116                    if (curr_cluster[k] in ref_cluster):117                        self.merge_clusters(ref_cluster,curr_cluster)118                        candidates.pop(run_index_j)119                        found = True120                        run_index_i = 0121                        break122                if (found):123                    break124                else:125                    run_index_j += 1126            if (not found):127                run_index_i += 1 128            129                130        zscore = mean + threshold*std131        for i in range(len(candidates)):132            arr = candidates[i]133            cluster_info = self.find_pivot_subgraph(arr[0],arr,matrix,zscore,strict_cluster = False)134            cluster_dict["clusters"].append(cluster_info)135        return  {}136 137    def overlapped_clustering(self,matrix,embeddings,threshold,mean,std,cluster_dict):138        picked_dict = {}139        overlap_dict = {}140    141        zscore = mean + threshold*std142        for i in range(len(embeddings)):143            if (i in picked_dict):144                continue145            arr = self.get_terms_above_threshold(matrix,embeddings,i,zscore)146            cluster_info = self.find_pivot_subgraph(i,arr,matrix,zscore,strict_cluster = True)147            self.update_picked_dict(picked_dict,cluster_info["neighs"])148            self.update_overlap_stats(overlap_dict,cluster_info)149            cluster_dict["clusters"].append(cluster_info)150        sorted_d = self.bucket_overlap(overlap_dict)151        return  sorted_d152        153        154    def cluster(self,output_file,texts,embeddings,threshold,clustering_type):155        is_overlapped = True if clustering_type == "overlapped" else False156        matrix = self.compute_matrix(embeddings)157        mean = np.mean(matrix)158        std = np.std(matrix)159        zscores = []160        inc = 0161        value = mean162        while (value < 1):163            zscores.append({"threshold":inc,"cosine":round(value,2)})164            inc += 1165            value = mean + inc*std166        #print("In clustering:",round(std,2),zscores)167        cluster_dict = {}168        cluster_dict["clusters"] = []169        if (is_overlapped):170            sorted_d = self.overlapped_clustering(matrix,embeddings,threshold,mean,std,cluster_dict) 171        else:172            sorted_d = self.non_overlapped_clustering(matrix,embeddings,threshold,mean,std,cluster_dict) 173        curr_threshold = f"{threshold} (cosine:{mean+threshold*std:.2f})"174        cluster_dict["info"] ={"mean":mean,"std":std,"current_threshold":curr_threshold,"zscores":zscores,"overlap":list(sorted_d.items())}175        return cluster_dict176 177 178