Linhz/ViMNer
1
1import codecs
2import numpy as np
3
4
5def get_chunks(seq, tags):
6 """
7 tags:dic{'per':1,....}
8 Args:
9 seq: [4, 4, 0, 0, ...] sequence of labels
10 tags: dict["O"] = 4
11 Returns:
12 list of (chunk_type, chunk_start, chunk_end)
13
14 Example:
15 seq = [4, 5, 0, 3]
16 tags = {"B-PER": 4, "I-PER": 5, "B-LOC": 3}
17 result = [("PER", 0, 2), ("LOC", 3, 4)]
18 """
19 default = tags['O']
20 idx_to_tag = {idx: tag for tag, idx in tags.items()}
21 chunks = []
22 # chunk_type用于判断是什么类型，LOC,PER
23 chunk_type, chunk_start = None, None
24 for i, tok in enumerate(seq):
25 # End of a chunk 1
26 if tok == default and chunk_type is not None:
27 # Add a chunk.
28 chunk = (chunk_type, chunk_start, i)
29 chunks.append(chunk)
30 chunk_type, chunk_start = None, None
31
32 # End of a chunk + start of a chunk!
33 elif tok != default:
34 # tok_chunk_class 判断是以B开头还是I开头
35 # tok_chunk_type 判断是什么类型，PER,LOC
36 tok_chunk_class, tok_chunk_type = get_chunk_type(tok, idx_to_tag)
37 if chunk_type is None:
38 chunk_type, chunk_start = tok_chunk_type, i
39 elif tok_chunk_type != chunk_type or tok_chunk_class == "B":
40 chunk = (chunk_type, chunk_start, i)
41 chunks.append(chunk)
42 chunk_type, chunk_start = tok_chunk_type, i
43 else:
44 pass
45 # end condition
46 if chunk_type is not None:
47 chunk = (chunk_type, chunk_start, len(seq))
48 chunks.append(chunk)
49 return chunks
50
51
52def get_chunk_type(tok, idx_to_tag):
53 """
54 Args:
55 tok: id of token, such as 4
56 idx_to_tag: dictionary {4: "B-PER", ...}
57 Returns:
58 tuple: "B", "PER"
59 """
60 tag_name = idx_to_tag[tok]
61 tag_class = tag_name.split('-')[0]
62 tag_type = tag_name.split('-')[-1]
63 return tag_class, tag_type
64
65
66# def run_evaluate(self, sess, test, tags):
67def evaluate(labels_pred, labels, words, tags):
68 """
69 words,pred, right: is a sequence, is label index or word index.
70 Evaluates performance on test set
71 Args:
72 sess: tensorflow session
73 test: dataset that yields tuple of sentences, tags
74 tags: {tag: index} dictionary
75 Returns:
76 accuracy
77 f1 score
78 ...
79 """
80
81 # file_write = open('./test_results.txt','w')
82
83 index = 0
84 sents_length = []
85
86 accs = []
87 correct_preds, total_correct, total_preds = 0., 0., 0.
88
89 for lab, lab_pred, word_sent in zip(labels, labels_pred, words):
90 word_st = word_sent
91 lab = lab
92 lab_pred = lab_pred
93 accs += [a == b for (a, b) in zip(lab, lab_pred)]
94 lab_chunks = set(get_chunks(lab, tags))
95 lab_pred_chunks = set(get_chunks(lab_pred, tags))
96 correct_preds += len(lab_chunks & lab_pred_chunks)
97 total_preds += len(lab_pred_chunks)
98 total_correct += len(lab_chunks)
99
100 # for i in range(len(word_st)):
101 # file_write.write('%s\t%s\t%s\n'%(word_st[i],lab[i],lab_pred[i]))
102 # file_write.write('\n')
103
104 p = correct_preds / total_preds if correct_preds > 0 else 0
105 r = correct_preds / total_correct if correct_preds > 0 else 0
106 f1 = 2 * p * r / (p + r) if correct_preds > 0 else 0
107 acc = np.mean(accs)
108
109 # file_write.close()
110 return acc, f1, p, r
111
112
113def evaluate_each_class(labels_pred, labels, words, tags, class_type):
114 # class_type:PER or LOC or ORG
115 index = 0
116
117 accs = []
118 correct_preds, total_correct, total_preds = 0., 0., 0.
119 correct_preds_cla_type, total_preds_cla_type, total_correct_cla_type = 0., 0., 0.
120
121 for lab, lab_pred, word_sent in zip(labels, labels_pred, words):
122 lab_pre_class_type = []
123 lab_class_type = []
124
125 word_st = word_sent
126 lab = lab
127 lab_pred = lab_pred
128 lab_chunks = get_chunks(lab, tags)
129 lab_pred_chunks = get_chunks(lab_pred, tags)
130 for i in range(len(lab_pred_chunks)):
131 if lab_pred_chunks[i][0] == class_type:
132 lab_pre_class_type.append(lab_pred_chunks[i])
133 lab_pre_class_type_c = set(lab_pre_class_type)
134
135 for i in range(len(lab_chunks)):
136 if lab_chunks[i][0] == class_type:
137 lab_class_type.append(lab_chunks[i])
138 lab_class_type_c = set(lab_class_type)
139
140 lab_chunksss = set(lab_chunks)
141 correct_preds_cla_type += len(lab_pre_class_type_c & lab_chunksss)
142 total_preds_cla_type += len(lab_pre_class_type_c)
143 total_correct_cla_type += len(lab_class_type_c)
144
145 p = correct_preds_cla_type / total_preds_cla_type if correct_preds_cla_type > 0 else 0
146 r = correct_preds_cla_type / total_correct_cla_type if correct_preds_cla_type > 0 else 0
147 f1 = 2 * p * r / (p + r) if correct_preds_cla_type > 0 else 0
148
149 return f1, p, r
150
151
152if __name__ == '__main__':
153 max_sent = 10
154 tags = {'0': 0,
155 'B-PER': 1, 'I-PER': 2,
156 'B-LOC': 3, 'I-LOC': 4,
157 'B-ORG': 5, 'I-ORG': 6,
158 'B-OTHER': 7, 'I-OTHER': 8,
159 'O': 9}
160 labels_pred = [
161 [9, 9, 9, 1, 3, 1, 2, 2, 0, 0],
162 [9, 9, 9, 1, 3, 1, 2, 0, 0, 0]
163 ]
164 labels = [
165 [9, 9, 9, 9, 3, 1, 2, 2, 0, 0],
166 [9, 9, 9, 9, 3, 1, 2, 2, 0, 0]
167 ]
168 words = [
169 [0, 0, 0, 0, 0, 3, 6, 8, 5, 7],
170 [0, 0, 0, 4, 5, 6, 7, 9, 1, 7]
171 ]
172 id_to_vocb = {0: 'a', 1: 'b', 2: 'c', 3: 'd', 4: 'e', 5: 'f', 6: 'g', 7: 'h', 8: 'i', 9: 'j'}
173 # new_words = []
174 # for i in range(len(words)):
175 # sent = []
176 # for j in range(len(words[i])):
177 # sent.append(id_to_vocb[words[i][j]])
178 # new_words.append(sent)
179 # class_type = 'PER'
180 # acc, f1,p,r = evaluate(labels_pred, labels,new_words,tags)
181 # print(p,r,f1)
182 # f1,p,r = evaluate_each_class(labels_pred, labels,new_words,tags, class_type)
183 # print(p,r,f1)
184
185 acc, f1, p, r = evaluate(labels_pred, labels, words, tags)
186 print(acc)