ZJF-Thunder/ChineseBert_text_analysis_system
0
1"""2功能：加载微博谣言数据集，微调预训练模型并以此进行判断输入文本是否为谣言3"""4import json5import os6import random7import zipfile8import jieba9import torch10import time11import datetime12import logging13import matplotlib.pyplot as plt14from matplotlib.animation import FuncAnimation15from torch.utils.data import DataLoader, TensorDataset16from transformers import BertTokenizer, BertForSequenceClassification17from transformers import get_linear_schedule_with_warmup, BertConfig18from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score19from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score20from sklearn.metrics import precision_recall_curve, roc_curve, average_precision_score, auc21from torchmetrics.functional import hamming_distance22 23 24# 解压原始数据集，将Rumor_Dataset.zip解压至data目录下25# 并解析出谣言和非谣言数据，并将所有数据写入all_data.txt26def data_preprocessing(src_path, target_path):27 """功能：28 解压原始数据集，将Rumor_Dataset.zip解压至data目录下，29 并解析出谣言和非谣言数据，并将所有数据写入all_data.txt30 31 :param src_path: 需要解压的文件的路径32 :param target_path: 解压之后的文件存放路径33 :return: 所有数据写入all_data.txt34 """35 if not os.path.isdir(target_path):36 z = zipfile.ZipFile(src_path, 'r')37 z.extractall(path=target_path)38 z.close()39 40 rumor_class_dirs = os.listdir(target_path + "/Chinese_Rumor_Dataset-master/CED_Dataset/rumor-repost/")41 non_rumor_class_dirs = os.listdir(target_path + "/Chinese_Rumor_Dataset-master/CED_Dataset/non-rumor-repost/")42 original_microblog = target_path + "/Chinese_Rumor_Dataset-master/CED_Dataset/original-microblog/"43 44 # 谣言为0，非谣言为145 rumor_label = "0"46 non_rumor_label = "1"47 # 谣言和非谣言的数量48 rumor_num = 049 non_rumor_num = 050 # 谣言和非谣言列表51 all_rumor_list = []52 all_non_rumor_list = []53 54 # 解析谣言数据55 for rumor_class_dir in rumor_class_dirs:56 if rumor_class_dir != '.DS_Store':57 with open(original_microblog + rumor_class_dir, 'r', encoding='utf-8') as f:58 rumor_content = f.read()59 rumor_dict = json.loads(rumor_content)60 all_rumor_list.append(rumor_label + "\t" + rumor_dict["text"] + "\n")61 rumor_num += 162 63 # 解析非谣言数据64 for non_rumor_class_dir in non_rumor_class_dirs:65 if non_rumor_class_dir != '.DS_Store':66 with open(original_microblog + non_rumor_class_dir, 'r', encoding='utf-8') as f2:67 non_rumor_content = f2.read()68 non_rumor_dict = json.loads(non_rumor_content)69 all_non_rumor_list.append(non_rumor_label + "\t" + non_rumor_dict["text"] + "\n")70 non_rumor_num += 171 72 print("谣言数据总量为：" + str(rumor_num))73 print("非谣言数据总量为：" + str(non_rumor_num))74 logging.info("谣言数据总量为：" + str(rumor_num))75 logging.info("非谣言数据总量为：" + str(non_rumor_num))76 # 将谣言列表和非谣言列表连接成一个新的列表77 all_data_list = all_rumor_list + all_non_rumor_list78 # 将文本打乱顺序79 random.shuffle(all_data_list)80 # 返回随机打乱的数据列表81 all_data_path = "./data/all_data.txt"82 all_rumor_path = "./data/rumor_data.txt"83 all_non_rumor_path = "./data/non_rumor_data.txt"84 # 清空文本的数据，并往all_data_path的txt文本中写入所有文本数据85 with open(all_data_path, 'w') as f:86 f.seek(0)87 f.truncate()88 with open(all_data_path, 'a', encoding='utf-8') as f:89 for data in all_data_list:90 f.write(data)91 # 清空文本的数据，并往all_rumor_path的txt文本中写入所有谣言文本数据92 with open(all_rumor_path, 'w') as f:93 f.seek(0)94 f.truncate()95 with open(all_rumor_path, 'a', encoding='utf-8') as f:96 for data in all_rumor_list:97 f.write(data)98 # 清空文本的数据，并往all_non_rumor_path的txt文本中写入所有非谣言文本数据99 with open(all_non_rumor_path, 'w') as f:100 f.seek(0)101 f.truncate()102 with open(all_non_rumor_path, 'a', encoding='utf-8') as f:103 for data in all_non_rumor_list:104 f.write(data)105 print("数据文本生成成功！")106 logging.info("数据文本生成成功！")107 108 109# 获取谣言和非谣言的数据条数110def get_rumor_norumor_num(data_path):111 """功能：112 获取谣言和非谣言的数据条数113 114 :param data_path: 扩充后的数据集地址115 :return: 无116 """117 rumor_count = 0118 nonrumor_count = 0119 with open(data_path, 'r', encoding='utf-8') as f:120 lines = f.readlines()121 for line in lines:122 label, text = line.strip().split('\t')123 if label == '0':124 rumor_count += 1125 else:126 nonrumor_count += 1127 print(f"数据集总数：{len(lines)} 条")128 print(f"谣言总数：{rumor_count} 条")129 print(f"非谣言总数：{nonrumor_count} 条")130 logging.info(f"数据集总数：{len(lines)} 条")131 logging.info(f"谣言总数：{rumor_count} 条")132 logging.info(f"非谣言总数：{nonrumor_count} 条")133 134 135# 按照特定比例划分数据集，默认比例为9：1136def splitting_dataset(data_path, train_size=0.9):137 """功能：138 按照特定比例划分数据集，默认比例为9：1139 140 :param data_path: 需要划分的数据的地址141 :param train_size: 训练集的占比大小，默认为0.9142 :return: 训练集和测试集的数据地址143 train_data_path = './data/train_data.txt'144 test_data_path = './data/test_data.txt'145 """146 with open(data_path, 'r', encoding='utf-8') as f:147 data = f.readlines()148 random.shuffle(data)149 train_data = data[:int(len(data) * train_size)]150 test_data = data[int(len(data) * train_size):]151 train_data_path = './data/train_data.txt'152 test_data_path = './data/test_data.txt'153 with open(train_data_path, 'w', encoding='utf-8') as f:154 f.writelines(train_data)155 with open(test_data_path, 'w', encoding='utf-8') as f:156 f.writelines(test_data)157 print("成功划分训练集和测试集！")158 logging.info("成功划分训练集和测试集！")159 print(f"训练集：{len(train_data)} 条")160 print(f"测试集：{len(test_data)} 条")161 logging.info(f"训练集：{len(train_data)} 条")162 logging.info(f"测试集：{len(test_data)} 条")163 164 return train_data_path, test_data_path165 166 167# 读取数据并将其转换为模型所需的格式168def read_data(filename, tokenizer, max_seq_length=256):169 """功能：170 读取数据并将其转换为模型所需的格式171 172 :param filename: 需要转换的数据的路径173 :param tokenizer: 模型的分词器174 :param max_seq_length: 最长文本长度，大于则截断，小于则填充175 :return: 转换成模型所需格式的数据176 """177 with open(filename, 'r', encoding='utf-8') as f:178 lines = f.readlines()179 180 data = []181 for line in lines:182 label, text = line.strip().split('\t')183 # 使用jieba进行高级别的精确分词，并去除停用词184 words = jieba.lcut(text, cut_all=False)185 text = ' '.join(words) # 将分词结果用空格拼接186 # 这里面涉及到分词的操作其实是encod_plus调用了tokenizer.tokenize函数来进行分词187 inputs = tokenizer.encode_plus(188 text,189 add_special_tokens=True, # 添加特殊标记[CLS]和[SEP]到序列190 padding='max_length', # 将序列填充到最大长度191 truncation=True, # 如果序列超过最大长度，则将其截断到最大长度192 max_length=max_seq_length, # 指定编码序列的最大长度193 mask_type='word', # 启用全词掩码策略194 mask_prob=0.15, # 指定遮盖比例为15%195 return_tensors='pt', # 返回PyTorch张量196 is_split_into_words=True)197 """"198 add_special_tokens = True将特殊标记如[CLS]和[SEP]添加到输入序列的开头和结尾。199 padding = 'max_length' 将序列填充到最大长度。200 truncation = True 如果序列超过最大长度，则将其截断到最大长度。201 max_length = 256 指定编码序列的最大长度。202 return_attention_mask = True返回一个注意力掩码，指示哪些标记是填充标记。203 return_token_type_ids = True返回标记类型ID，指示哪些标记属于哪个段落（对于问答等任务很有用）。204 return_tensors = 'pt' 返回PyTorch张量而不是列表。205 """206 data.append((inputs.input_ids, inputs.token_type_ids, inputs.attention_mask, int(label)))207 # 将数据转换为PyTorch张量,并在GPU上计算208 """改进版本:下面两种方式都可以,不会警告"""209 # data_tensor = [210 # (torch.as_tensor(input_ids).cuda(), torch.as_tensor(token_type_ids).cuda(),211 # torch.as_tensor(attention_mask).cuda(), torch.tensor([[label]]).cuda())212 # for input_ids, token_type_ids, attention_mask, label in data]213 data_tensor = [214 (input_ids.clone().clone().detach().cuda(), token_type_ids.clone().clone().detach().cuda(),215 attention_mask.clone().clone().detach().cuda(),216 torch.tensor([[label]]).cuda()) for input_ids, token_type_ids, attention_mask, label in data]217 return data_tensor218 219 220# 对标签进行独热编码221def one_hot_encoding(label_ids, num_labels):222 """功能：对标签进行独热编码223 224 :param label_ids: 标签，一般是个list或者tensor225 :param num_labels: 标签数量226 :return: 独热编码之后的标签227 """228 labels = torch.zeros((len(label_ids), num_labels))229 labels[torch.arange(len(label_ids)), label_ids] = 1230 return labels231 232 233# 训练模型234def model_train(model, train_tensor):235 """功能：236 输入训练数据，训练(微调)模型237 238 :param model: 定义的预训练模型239 :param train_tensor: 输入的训练数据，数据类型为tensor240 :return: 训练(微调)好的模型241 """242 train_tensor_stack = [torch.stack(tensors) for tensors in zip(*train_tensor)]243 train_dataset = TensorDataset(*train_tensor_stack)244 """数据加载器，一个Batch(批次)等于样本总数/Batch_size245 例：训练数据一共3048个，batch_size为16，所以一共3048/16个batch，取整为190个batch246 所以train_loader的长度为190，它的作用是将3040个样本分割为190个batch，用于训练"""247 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=Batch_size, drop_last=True)248 249 # 将数据加载到模型中进行微调250 optimizer = torch.optim.Adam(model.parameters(), lr=Learning_rate)251 """252 # 定义交叉熵损失函数 253 # criterion1 = torch.nn.CrossEntropyLoss()254 # 定义二分类交叉损失函数255 # criterion2 = torch.nn.BCEWithLogitsLoss()256 """257 # 增加学习率动态调整机制258 total_steps = len(train_loader) * Epochs259 scheduler = get_linear_schedule_with_warmup(optimizer,260 num_warmup_steps=0, # 预热步数261 num_training_steps=total_steps) # 总的训练步数262 # 将模型移动到GPU设备263 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")264 model.to(device)265 model.train()266 267 # 记录每一次迭代的损失268 train_losses = []269 print("————————————————模型训练开始————————————————")270 logging.info("————————————————模型训练开始————————————————")271 # 训练开始时间272 start_time = time.time()273 for epoch in range(Epochs):274 # 每一个Epoch的平均loss275 epoch_loss = 0276 # 迭代一次就是迭代一个batch(批次)277 for batch, data in enumerate(train_loader):278 # data = (input_ids, token_type_ids, attention_mask, labels)279 # 正向传播280 input_ids = data[0].to(device).squeeze(1)281 token_type_ids = data[1].to(device).squeeze(1)282 attention_mask = data[2].to(device).squeeze(1)283 labels = data[3].to(device).squeeze() # 形状为1维：（16，）284 # 将标签进行独热编码，变成长度为2的向量，和模型的输出层的长度一样，然后输入到模型中，并且调用二分类交叉损失函数285 labels = one_hot_encoding(labels, Num_labels)286 labels = labels.cuda()287 # 将模型的参数梯度清零，以免上一个epoch的梯度对当前epoch的训练产生影响288 optimizer.zero_grad()289 # 前向传播290 291 outputs = model(input_ids,292 token_type_ids=token_type_ids,293 attention_mask=attention_mask,294 labels=labels)295 296 """调用模型自带的损失函数,297 如果是二分类任务，模型会自动调用二分类交叉熵损失函数计算损失298 直接loss = outputs.loss调用即可"""299 loss = outputs.loss300 """301 # 调用自定义的交叉熵损失函数计算loss，这时候label不需要进行独热编码302 # loss = criterion1(outputs.logits, labels)303 # 调用自定义的二分类交叉熵损失函数计算损失,labels进行独热编码之后,本身就是浮点数,所以加不加float都一样304 # loss = criterion2(outputs.logits, labels.float())305 """306 # 反向传播307 loss.backward()308 # 参数更新309 optimizer.step()310 # 调整学习率311 scheduler.step()312 # 记录一个Batch的loss，用以计算一个Epoch的平均loss313 epoch_loss += loss.item()314 train_losses.append(loss.item())315 # 每十个批次输出一次训练信息316 if batch % 10 == 0:317 print('Epoch: {}, Batch: {}, Loss: {}'.format(epoch, batch, loss.item()))318 logging.info('Epoch: {}, Batch: {}, Loss: {}'.format(epoch, batch, loss.item()))319 avg_epoch_loss = epoch_loss / len(train_loader)320 print("Epoch [{}/{}], Average Loss: {:.4f}".format(epoch + 1, Epochs, avg_epoch_loss))321 logging.info("Epoch [{}/{}], Average Loss: {:.4f}".format(epoch + 1, Epochs, avg_epoch_loss))322 323 end_time = time.time()324 total_time = end_time - start_time325 print("————————————————模型训练完成————————————————")326 print("训练开始时间：", time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time)))327 print("训练结束时间：", time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end_time)))328 print(f"模型训练总时间为: {total_time:.2f} 秒")329 logging.info("————————————————模型训练完成————————————————")330 logging.info("训练开始时间：" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time)))331 logging.info("训练结束时间：" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end_time)))332 logging.info(f"模型训练总时间为: {total_time:.2f} 秒")333 # 获取当前时间334 local_time = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime(end_time))335 save_dir = './模型保存' # 保存模型的相对路径336 if not os.path.exists(save_dir):337 os.makedirs(save_dir) # 如果目录不存在，创建目录338 model_name = os.path.join(save_dir, "ChineseBert_{}.pt".format(local_time)) # 模型的相对路径339 torch.save(model.state_dict(), model_name)340 print("————————————————模型保存成功————————————————")341 logging.info("————————————————模型保存成功————————————————")342 343 # 绘制并保存训练损失的静态图344 # 获取当前日期345 today = datetime.date.today()346 date_str = today.strftime('%Y-%m-%d')347 # 获取当前具体时间348 local_time = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime())349 # 创建日志文件路径350 img_save_dir = os.path.join('./images/training_loss/', f"{date_str}_Training_loss")351 # 判断文件夹是否存在，如果不存在则创建352 if not os.path.exists(img_save_dir):353 os.makedirs(img_save_dir)354 print(f"文件夹 {img_save_dir} 创建成功！")355 356 else:357 print(f"文件夹 {img_save_dir} 已存在！")358 359 plt.figure(figsize=(10, 5))360 # 散点图361 # plt.scatter(range(len(train_losses)), train_losses)362 # 折线图363 plt.clf() # 首先清空当前图像364 plt.plot(range(len(train_losses)), train_losses, scalex=10, label='Training Loss')365 plt.legend(loc='upper right') # 添加图例366 plt.title("Training Loss")367 plt.xlabel("Batch")368 plt.ylabel("Loss")369 images = os.path.join(img_save_dir, "training_loss_{}.png".format(local_time))370 plt.savefig(images)371 372 # 绘制训练损失的动态图373 show_animation(train_losses)374 375 return model376 377 378# 展示训练损失的动态过程图379def show_animation(train_losses):380 """功能：展示训练损失的动态过程图381 382 :param train_losses: 训练的总损失383 :return: 无384 """385 # 获取当前日期386 today = datetime.date.today()387 date_str = today.strftime('%Y-%m-%d')388 # 获取当前具体时间389 local_time = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime())390 # 创建日志文件路径391 img_save_dir = os.path.join('./images/training_loss/', f"{date_str}_Training_loss")392 # 判断文件夹是否存在，如果不存在则创建393 if not os.path.exists(img_save_dir):394 os.makedirs(img_save_dir)395 print(f"文件夹 {img_save_dir} 创建成功！")396 else:397 print(f"文件夹 {img_save_dir} 已存在！")398 399 fig, ax = plt.subplots()400 ax.set(xlim=(0, len(train_losses)), ylim=(0, 1))401 402 # 定义内部函数，用于保存动画完成后最后完整的图像403 def save_final_frame(train_losses, fig, ax):404 ax.clear()405 ax.plot(range(len(train_losses)), train_losses, label='Training Loss')406 ax.legend(loc='upper right')407 ax.set(xlabel='Batch', ylabel='Loss', title='Training Loss')408 if not hasattr(save_final_frame, 'saved'):409 images = os.path.join(img_save_dir, "training_loss_{}.png".format(local_time))410 # fig.savefig(images) # 如果不保存，则注释掉这句411 # 设置 saved 标志为 True，表示已经保存最终的图像412 save_final_frame.saved = True413 414 # 定义在每一帧调用的动画函数415 def animate(i):416 ax.clear()417 ax.plot(range(i + 1), train_losses[:i + 1], label='Training Loss')418 ax.legend(loc='upper right')419 ax.set(xlabel='Batch', ylabel='Loss', title='Training Loss')420 plt.pause(0.1)421 422 ani = FuncAnimation(fig, animate, frames=len(train_losses), repeat=False)423 ani.event_source.add_callback(save_final_frame, train_losses=train_losses, fig=fig, ax=ax)424 plt.show() # 注释打开则播放训练损失动画425 426 427# 定义性能指标函数428def compute_metrics(y_true, y_pred):429 """功能：定义计算模型性能指标函数430 431 :param y_true: 真实的标签列表432 :param y_pred: 预测的标签列表433 :return:返回各项计算结果434 """435 # 准确率、准确度436 accuracy = accuracy_score(y_true, y_pred)437 # 精确度、精度：所有被分类器正确分类的正样本占所有被分类器分类为正样本的样本数的比例438 precision = precision_score(y_true, y_pred, average='macro')439 # 召回率：被正确分类的正样本占所有实际正样本的比例440 recall = recall_score(y_true, y_pred, average='macro')441 # F1：精确度和召回率的调和平均值442 f1 = f1_score(y_true, y_pred, average='macro')443 # AUC值：ROC曲线下的面积,取值范围在0到1之间，越接近1代表模型的性能越好444 auc = roc_auc_score(y_true, y_pred)445 result = {'accuracy': accuracy,446 'precision': precision,447 'recall': recall,448 'f1_score': f1,449 'AUC': auc}450 return result451 452 453# 测试模型454def model_eval(model, test_tensor):455 """功能：测试训练好的模型的性能456 457 :param model: 训练(微调)好的模型458 :param test_tensor: 输入的测试数据，数据类型为tensor459 :return: 无460 """461 # 在测试集上进行评估462 test_tensor_stack = [torch.stack(tensors) for tensors in zip(*test_tensor)]463 test_dataset = TensorDataset(*test_tensor_stack)464 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=Batch_size, drop_last=True)465 # 将模型移动到GPU设备466 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")467 model.to(device)468 model.eval()469 # print("模型信息：", model)470 correct = 0471 total = 0472 # 存放真实标签和预测标签473 label_true = []474 label_pred = []475 print("————————————————模型测试开始————————————————")476 logging.info("————————————————模型测试开始————————————————")477 with torch.no_grad():478 for i, data in enumerate(test_loader):479 # data = (input_ids, token_type_ids, attention_mask, labels)480 # 正向传播481 input_ids = data[0].to(device).squeeze(1) # 降维度482 token_type_ids = data[1].to(device).squeeze(1)483 attention_mask = data[2].to(device).squeeze(1)484 labels = data[3].to(device).squeeze()485 print("输入实际标签为：", labels)486 logging.info("输入实际标签为：" + str(labels.tolist()))487 488 outputs = model(input_ids,489 token_type_ids=token_type_ids,490 attention_mask=attention_mask)491 """492 # 这里不能加item()来取出预测值，因为验证的时候是用一个Batch_size大小的样本同时预测的，一个批次就是16个样本493 # 这里predictiongs不是一个样本，是一个有着16个样本预测标签的类似列表或者张量494 predictions = (torch.sigmoid(outputs.logits) >= 0.5).int()[:, 1] # 使用阈值0.5将概率转为二元标签495 predictions = torch.argmax(torch.sigmoid(outputs.logits), dim=1).cuda()496 """497 predictions = torch.argmax(torch.softmax(outputs.logits, dim=1), dim=1).cuda()498 print("输入预测标签为：", predictions)499 logging.info("输入预测标签为：" + str(predictions.cuda().tolist()))500 501 correct += (predictions == labels).sum().item()502 total += labels.size(0)503 504 # sklearn的指标 全部tensor转换成list505 """extend和append的区别：506 extend是直接在后面追加所有元素，如果追加的是一个列表，则是直接追加这个列表的所有值507 append是在后面追加一个单独的元素，这个元素可以是一个列表或者一个值"""508 label_true.extend(labels.cuda().tolist())509 label_pred.extend(predictions.cuda().tolist())510 511 print("————————————————模型测试结束————————————————")512 logging.info("————————————————模型测试结束————————————————")513 accuracy = correct / total514 print("总测试样本：{}个".format(total))515 print("预测正确样本:{}个".format(correct))516 print('测试准确度:', accuracy)517 logging.info('总测试样本: %d个', total)518 logging.info('预测正确样本: %d个', correct)519 logging.info('测试准确度: %f', accuracy)520 521 # sklearn的指标522 metrics = compute_metrics(label_true, label_pred)523 print("-----------------------------")524 print(f"Accuracy: {metrics['accuracy']}")525 print(f"Precision: {metrics['precision']}")526 print(f"Recall: {metrics['recall']}")527 print(f"F1 Score: {metrics['f1_score']}")528 print(f"AUC Score: {metrics['AUC']}")529 print("-----------------------------")530 print(classification_report(label_true, label_pred))531 print("-----------------------------")532 print("-----------混淆矩阵------------")533 print(confusion_matrix(label_true, label_pred))534 print("-----------------------------")535 536 logging.info("-----------------------------")537 logging.info(f"Accuracy: {metrics['accuracy']}")538 logging.info(f"Precision: {metrics['precision']}")539 logging.info(f"Recall: {metrics['recall']}")540 logging.info(f"F1 Score: {metrics['f1_score']}")541 logging.info(f"AUC Score: {metrics['AUC']}")542 logging.info("-----------------------------")543 logging.info("\n" + classification_report(label_true, label_pred))544 logging.info("-----------------------------")545 logging.info("-----------混淆矩阵------------")546 logging.info("\n" + str(confusion_matrix(label_true, label_pred)))547 logging.info("-----------------------------")548 549 # 计算average_precision_score的值550 average_precision = average_precision_score(label_true, label_pred)551 print('Average precision-recall score: {0:0.2f}'.format(average_precision))552 # 计算汉明损失553 hamming_loss = hamming_distance(torch.tensor(label_pred), torch.tensor(label_true))554 print(f"Hamming Loss: {hamming_loss}")555 556 # 计算不同概率阈值的精度-召回率对557 precision, recall, thresholds = precision_recall_curve(label_true, label_pred)558 # 计算AUC-PR559 auc_PR = auc(recall, precision)560 # 绘制PR曲线:纵轴是精度（precision），横轴是召回率（recall）561 plt.clf() # 首先清空当前图像562 plt.plot(recall, precision, lw=2, color='blue', label='AUC-PR = %0.4f' % auc_PR)563 plt.xlim([0.0, 1.0])564 plt.ylim([0.0, 1.05])565 plt.xlabel('Recall')566 plt.ylabel('Precision')567 plt.title('PR Curve')568 plt.legend(loc="lower right")569 img_save_dir = './images/PR曲线'570 if not os.path.exists(img_save_dir):571 os.makedirs(img_save_dir)572 # 获取当前时间573 local_time = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime(time.time()))574 images = os.path.join(img_save_dir, "PR_{}.png".format(local_time))575 plt.savefig(images)576 # plt.show()577 578 # 计算假阳性率（FPR）和真阳性率（TPR）和阈值（thresholds）579 FPR, TPR, thresholds = roc_curve(label_true, label_pred)580 # 计算AUC-ROC581 auc_ROC = auc(FPR, TPR)582 # 绘制ROC曲线:纵轴是真阳率（true positive rate），横轴是假阳率（false positive rate）583 plt.clf() # 首先清空当前图像584 plt.plot(FPR, TPR, lw=2, color='red', label='AUC-ROC = %0.4f' % auc_ROC)585 plt.xlim([0.0, 1.0])586 plt.ylim([0.0, 1.05])587 plt.xlabel('False Positive Rate')588 plt.ylabel('True Positive Rate')589 plt.title('ROC Curve')590 plt.legend(loc="lower right")591 img_save_dir = './images/ROC曲线'592 if not os.path.exists(img_save_dir):593 os.makedirs(img_save_dir)594 # 获取当前时间595 local_time = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime(time.time()))596 images = os.path.join(img_save_dir, "ROC_{}.jpg".format(local_time))597 plt.savefig(images)598 # plt.show()599 600 601# 定义预测函数602def predicted(text, model, tokenizer):603 """功能：输入一条文本，调用模型，返回预测结果604 605 :param text: 输入的文本，类型为str606 :param model: 训练好的模型607 :param tokenizer: 预训练模型的分词器608 :return: 返回预测的标签和概率609 """610 words = jieba.lcut(text, cut_all=False)611 text = ' '.join(words) # 将分词结果用空格拼接612 """613 # 将文本转换成数字编码，这里是直接调用tokenizer类，614 # 其实内部处理分词的函数还是tokenizer.tokenize()615 # 并且内部encode的函数也是tokenizer.encode_plus()函数616 # input_ids = tokenizer(text,617 # padding=True,618 # truncation=True,619 # max_length=Max_seq_length,620 # return_tensors='pt').input_ids.cuda()621 """622 # 这里可以直接调用tokenizer.encode_plus()函数来将token变成token id623 inputs = tokenizer.encode_plus(text,624 padding=True,625 truncation=True,626 max_length=Max_seq_length,627 return_tensors='pt',628 is_split_into_words=True)629 input_ids = inputs['input_ids'].cuda()630 631 model.cuda()632 # 预测类别633 with torch.no_grad():634 outputs = model(input_ids)635 # 使用softmax将输出转换成概率分布636 probs = torch.softmax(outputs.logits, dim=1)637 # 输出概率列表：probs.tolist()[0]638 # 获取最大概率对应的类别标签及其对应的概率639 pred_label = torch.argmax(probs, dim=1).item()640 pred_prob = probs[0, pred_label].item()641 642 return pred_label, pred_prob643 644 645# 调用预测模型来预测文本类别646def Text_predict(trained_model, tokenizer):647 """功能：调用预测模型来预测文本类别，展示结果648 649 :param trained_model: 训练好的模型650 :param tokenizer: 预训练模型的分词器651 :return: 无652 """653 print("————————————————预测文本类别————————————————")654 logging.info("————————————————预测文本类别————————————————")655 text = '有着800多年历史的克里姆林宫，再次迎来中国国家主席。当地时间3月21日下午，' \656 '俄罗斯总统普京在大克里姆林宫二层乔治大厅为习近平主席举行隆重的欢迎仪式，随后两国元首举行会谈'657 predicted_label, predicted_prob = predicted(text, trained_model, tokenizer)658 # 另一个样本的标签及其概率659 other_prob = 1 - predicted_prob660 other_prob_label = 1661 # 定义标签662 labels = {0: "谣言", 1: "非谣言"}663 if predicted_label == 1:664 other_prob_label = 0665 print(f"这条微博有{predicted_prob * 100:.2f}%的概率为{labels[predicted_label]}，"666 f"有{other_prob * 100:.2f}%的概率为{labels[other_prob_label]}")667 logging.info(f"这条微博有{predicted_prob * 100:.2f}%的概率为{labels[predicted_label]}，"668 f"有{other_prob * 100:.2f}%的概率为{labels[other_prob_label]}")669 else:670 print(f"这条微博有{predicted_prob * 100:.2f}%的概率为{labels[predicted_label]}，"671 f"有{other_prob * 100:.2f}%的概率为{labels[other_prob_label]}")672 logging.info(f"这条微博有{predicted_prob * 100:.2f}%的概率为{labels[predicted_label]}，"673 f"有{other_prob * 100:.2f}%的概率为{labels[other_prob_label]}")674 675 676# 检查GPU是否可用，不可用则退出程序677def cuda_is_available():678 """679 功能：检查GPU是否可用，不可用则退出程序680 """681 if torch.cuda.is_available():682 print('GPU is available')683 logging.info('GPU is available')684 else:685 print('GPU is not available')686 logging.info('GPU is not available')687 print("————————正在退出程序——————————")688 logging.info("————————正在退出程序——————————")689 exit(1)690 691 692# 日志配置693def log_config():694 """695 功能：配置日志696 """697 # 获取当前日期698 today = datetime.date.today()699 date_str = today.strftime('%Y-%m-%d')700 # 获取当前具体时间701 local_time = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime())702 # 创建日志文件路径703 log_file_path = os.path.join('./运行日志/', f"{date_str}日志")704 # 判断文件夹是否存在，如果不存在则创建705 if not os.path.exists(log_file_path):706 os.makedirs(log_file_path)707 print(f"文件夹 {log_file_path} 创建成功！")708 else:709 print(f"文件夹 {log_file_path} 已存在！")710 log_filename = '{}/output_{}.log'.format(log_file_path, local_time)711 # 配置日志712 logging.basicConfig(713 level=logging.INFO,714 format='%(asctime)s - %(levelname)s - %(message)s',715 handlers=[716 # logging.StreamHandler(), # 输出到终端717 logging.FileHandler(log_filename, mode='w', encoding='utf-8') # 输出到文件718 ]719 )720 721 722# 主函数入口723def main():724 log_config()725 # 输出日志726 logging.info('——————程序运行开始——————')727 logging.info("——————————————————————")728 # 输出训练超参数729 logging.info("——————输出训练参数——————")730 logging.info("Batch_size：%s", Batch_size)731 logging.info("Epochs：%s", Epochs)732 logging.info("Learning_rate：%s", Learning_rate)733 logging.info("max_seq_length：%s", Max_seq_length)734 logging.info("num_labels：%s", Num_labels)735 logging.info("——————输出数据信息——————")736 # 检查GPU是否可用737 cuda_is_available()738 # 待解压文件路径739 src_path = "./data/Rumor_Dataset.zip"740 # 解压之后文件路径741 target_path = "./data//Chinese_Rumor_Dataset-master"742 # 所有数据的txt文本路径743 all_data_path = "./data/all_data.txt"744 # 将数据集解压，并将json文件数据处理保存成txt文本 没有扩充之前的数据集745 # data_preprocessing(src_path, target_path)746 # 划分训练集和测试集747 # train_data, test_data = splitting_dataset(all_data_path, train_size=0.9)748 749 # 获取谣言和非谣言的数据条数750 get_rumor_norumor_num("./data/all_data3.txt")751 # 用扩充之后的数据集按照特定比例划分训练集和测试集，all_data2.txt为扩充之后的数据集752 train_data, test_data = splitting_dataset("./data/all_data3.txt", train_size=0.9)753 754 # 可以在这里指定字典的路径755 # tokenizer = BertTokenizer('gdrive/My Drive/Colab Notebooks/vocab.txt')756 # 定义预训练模型757 # bert = './models/chinese-roberta-wwm-ext'758 # bert = './models/bert-base-chinese'759 bert = './models/chinese-bert-wwm-ext'760 # 加载原始config文件761 config = BertConfig.from_pretrained(bert, num_labels=Num_labels)762 config.save_pretrained('my_chinesebert_config')763 """# 加载配置文件，可以在参数中修改配置文件764 # config = BertConfig.from_pretrained(bert, output_hidden_states=True, hidden_dropout_prob=0.2,765 # 也可以单独定义配置文件中的参数，以此来修改配置文件，然后保存766 # attention_probs_dropout_prob=0.2)767 # 如果修改了，就保存修改后的配置文件，如果没有修改参数则和原始参数相同"""768 769 # 用BertTokenizer来作为模型加载770 # 加载预训练的BERT模型和中文分词器，并返回一个BertTokenizer对象，可以用于对中文文本进行分词和编码771 tokenizer = BertTokenizer.from_pretrained(bert, num_classes=Num_labels)772 model = BertForSequenceClassification.from_pretrained(bert, config=config)773 774 # 读取训练集和测试集775 train_tensor = read_data(train_data, tokenizer, max_seq_length=Max_seq_length)776 test_tensor = read_data(test_data, tokenizer, max_seq_length=Max_seq_length)777 778 # # 训练模型779 trained_model = model_train(model, train_tensor)780 # # 测试模型781 # model_eval(trained_model, test_tensor)782 # # 预测函数783 # Text_predict(trained_model, tokenizer)784 785 """用于测试已有模型"""786 # trained_model_path = './models/chinese-bert-wwm-ext' # 未经过微调的原始模型1787 # trained_model_path = './models/bert-base-chinese' # 未经过微调的原始模型2788 # trained_model_path = './模型保存/chinesebert.pth' # 微调的最早期的模型789 # trained_model_path = './模型保存/ChineseBert_2023-03-29_16-27-07_0.949.pt'790 # trained_model_path = './模型保存/ChineseBert_2023-03-25_17-10-39_0.95.pt'791 # trained_model_path = './模型保存/ChineseBert_2023-04-05_16-30-38_0.9970.pt'792 trained_model_path = './模型保存/ChineseBert_2023-04-07_20-12-29_0.999.pt'793 794 # trained_model_path = './模型保存/ChineseBert_2023-04-12_15-02-05__chinese-roberta-wwm-ext_0.999.pt'795 model = BertForSequenceClassification.from_pretrained(trained_model_path, config=config)796 model_eval(model, test_tensor)797 Text_predict(model, tokenizer)798 799 logging.info("——————————————————————")800 logging.info('——————程序运行结束——————')801 802 803if __name__ == '__main__':804 # 设置训练超参数805 Batch_size = 12806 Epochs = 2 # 4807 Learning_rate = 2e-5808 Num_labels = 2809 Max_seq_length = 256810 main()811