CSharpCorner/CSharpAI
0
1from transformers.pipelines.image_segmentation import Predictions2from transformers import DistilBertForSequenceClassification, DistilBertTokenizer3import unidecode, re, unicodedata4from bs4 import BeautifulSoup5from urllib.request import urlopen6from urllib.parse import urlparse7from sklearn.metrics import confusion_matrix, accuracy_score8import torch.nn.functional as F9import gradio as gr10import torch11import nltk12import json13 14def check_by_url(txt_url):15 parsed_url = urlparse(txt_url)16 url = (f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rsplit('/', 1)[0]}/")17 print(url)18 19 new_data = []20 page = urlopen(url=url).read().decode("utf-8")21 soup = BeautifulSoup(page, "html.parser")22 title = soup.find("title").get_text()23 24 # remove punctuations from title25 def remove_punctuation(title):26 punctuationfree = "".join([i for i in title if i not in string.punctuation])27 return punctuationfree28 29 css_class_to_remove = ("dp-highlighter") # Replace with the CSS class you want to remove30 # Find <div> tags with the specified CSS class and remove their content31 div_tags = soup.find_all(["code", "pre"])32 for div_tag in div_tags:33 div_tag.clear()34 35 div_tags = soup.find_all("div", class_=css_class_to_remove)36 for div_tag in div_tags:37 div_tag.clear()38 39 # Fetch content of remaining tags40 content_with_style = ""41 p_tags_with_style = soup.find_all("p", style=True)42 for p_tag in p_tags_with_style:43 p_content = re.sub(r"\n", "", p_tag.get_text())44 content_with_style += p_content45 46 # Fetch content of <p> tags without style47 content_without_style = ""48 p_tags_without_style = soup.find_all("p", style=False)49 for p_tag in p_tags_without_style:50 p_content = re.sub(r"\n", "", p_tag.get_text())51 content_without_style += p_content52 53 # Replace Unicode characters in the content and remove duplicates54 normalized_content_with_style = re.sub(r"\s+", " ", content_with_style) # Remove extra spaces55 normalized_content_with_style = normalized_content_with_style.replace("\r", "") # Replace '\r' characters56 normalized_content_with_style = unicodedata.normalize("NFKD", normalized_content_with_style)57 normalized_content_with_style = unidecode.unidecode(normalized_content_with_style)58 59 normalized_content_without_style = re.sub(r"\s+", " ", content_without_style) # Remove extra spaces60 normalized_content_without_style = normalized_content_without_style.replace("\r", "") # Replace '\r' characters61 normalized_content_without_style = unicodedata.normalize("NFKD", normalized_content_without_style)62 normalized_content_without_style = unidecode.unidecode(normalized_content_without_style)63 64 normalized_content_with_style += normalized_content_without_style65 new_data = {"title": title, "content": normalized_content_with_style}66# return new_data67 68 model = DistilBertForSequenceClassification.from_pretrained(".")69 tokenizer = DistilBertTokenizer.from_pretrained(".")70 71 label_mapping = {1: "SFW", 0: "NSFW"} 72 test_encodings = tokenizer.encode_plus(73 title, 74 truncation=True, 75 padding=True, 76 max_length=512, 77 return_tensors="pt"78 )79 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")80 test_input_ids = test_encodings["input_ids"].to(device)81 test_attention_mask = test_encodings["attention_mask"].to(device)82 with torch.no_grad():83 model = model.to(device)84 model.eval()85 outputs = model(test_input_ids, attention_mask=test_attention_mask)86 logits = outputs.logits87 predicted_labels = torch.argmax(logits, dim=1)88 probabilities = F.softmax(logits, dim=1)89 confidence_score_title = torch.max(probabilities, dim=1).values.tolist() 90 predicted_label_title = label_mapping[predicted_labels.item()]91 92 test_encodings = tokenizer.encode_plus(93 normalized_content_with_style,94 truncation=True,95 padding=True,96 max_length=512,97 return_tensors="pt",98 )99 test_input_ids = test_encodings["input_ids"].to(device)100 test_attention_mask = test_encodings["attention_mask"].to(device)101 with torch.no_grad():102 outputs = model(test_input_ids, attention_mask=test_attention_mask)103 logits = outputs.logits104 predicted_labels = torch.argmax(logits, dim=1)105 probabilities = F.softmax(logits, dim=1)106 confidence_scores_content = torch.max(probabilities, dim=1).values.tolist()107 predicted_label_content = label_mapping[predicted_labels.item()]108 109 return (110 predicted_label_title,111 confidence_score_title,112 predicted_label_content,113 confidence_scores_content,114 new_data,115 #new1,116 )117 118label_mapping = {1: "SFW", 0: "NSFW"} # 1:True 0:false119 120def predict_2(txt_url, normalized_content_with_style):121 (122 predicted_label_title,123 confidence_score_title,124 predicted_label_content,125 confidence_scores_content,126 new_data, 127 ) = (None, None, None, None, None)128 129 predicted_label_text, confidence_score_text = None, None130 131 if txt_url.startswith("http://") or txt_url.startswith("https://"):132 (133 predicted_label_title,134 confidence_score_title,135 predicted_label_content,136 confidence_scores_content,137 new_data,138 ) = check_by_url(txt_url)139 elif txt_url.startswith(""):140 model = DistilBertForSequenceClassification.from_pretrained(".")141 tokenizer = DistilBertTokenizer.from_pretrained(".")142 143 test_encodings = tokenizer.encode_plus(144 normalized_content_with_style,145 truncation=True,146 padding=True,147 max_length=512,148 return_tensors="pt",149 )150 151 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")152 test_input_ids = test_encodings["input_ids"].to(device)153 test_attention_mask = test_encodings["attention_mask"].to(device)154 155 with torch.no_grad():156 model = model.to(device)157 model.eval()158 outputs = model(test_input_ids, attention_mask=test_attention_mask)159 logits = outputs.logits160 predicted_labels = torch.argmax(logits, dim=1)161 probabilities = F.softmax(logits, dim=1)162 confidence_score_text = torch.max(probabilities, dim=1).values.tolist()163 predicted_label_text = label_mapping[predicted_labels.item()]164 165 return (166 predicted_label_title,167 confidence_score_title,168 predicted_label_content,169 confidence_scores_content,170 new_data,171 predicted_label_text,172 confidence_score_text,173 #new,174 )175 176def word_by_word(txt_url, normalized_content_with_style):177 if txt_url.startswith("http://") or txt_url.startswith("https://") or txt_url.startswith(""):178 (179 predicted_label_title,180 confidence_score_title,181 predicted_label_content,182 confidence_scores_content,183 new_data,184 predicted_label_text,185 confidence_score_text,186 ) = predict_2(txt_url, normalized_content_with_style)187 188 model = DistilBertForSequenceClassification.from_pretrained(".")189 tokenizer = DistilBertTokenizer.from_pretrained(".") 190 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")191 model = model.to(device)192 model.eval()193 194 new_word={}195 content_words =[]196 words_2 =[] 197 if predicted_label_content=="NSFW" or predicted_label_text=="NSFW":198 if txt_url.startswith("http://") or txt_url.startswith("https://"):199 content_words = new_data['content'].split()200 else:201 words_2 = normalized_content_with_style.split()202 203 results = []204 for word in content_words or words_2 : 205 encoding = tokenizer.encode_plus(206 word,207 truncation=True,208 padding=True,209 max_length=512,210 return_tensors="pt"211 )212 input_ids = encoding["input_ids"].to(device)213 attention_mask = encoding["attention_mask"].to(device)214 with torch.no_grad():215 outputs = model(input_ids, attention_mask=attention_mask)216 logits = outputs.logits217 probabilities = F.softmax(logits, dim=1)218 predicted_label = torch.argmax(logits, dim=1).item()219 #label_mapping = {1: "SFW", 0: "NSFW"} # 1:True 0:False220 predicted_label_word = label_mapping[predicted_label]221 confidence_score_word = torch.max(probabilities, dim=1).values.item()222 223 #new_word={} 224 if predicted_label_word=="NSFW":225 result = {"Word": word, "Label": predicted_label_word, "Confidence": confidence_score_word}226 results.append(result)227 new_word = json.dumps(results) 228 return(229 predicted_label_title,230 confidence_score_title,231 predicted_label_content,232 confidence_scores_content,233 new_data,234 predicted_label_text,235 confidence_score_text,236 new_word,237 )238 239 240demo = gr.Interface(241 fn=word_by_word,242 inputs=[243 gr.inputs.Textbox(label="URL", placeholder="Enter URL"),244 gr.inputs.Textbox(label="Text", placeholder="Enter Text"),245 ],246 outputs=[247 gr.outputs.Textbox(label="Title_prediction"),248 gr.outputs.Textbox(label="Title_confidence_score"),249 gr.outputs.Textbox(label="Content_prediction"),250 gr.outputs.Textbox(label="Content_confidence_score"),251 gr.outputs.Textbox(label="Description").style(show_copy_button=True),252 gr.outputs.Textbox(label="Text_prediction_score"),253 gr.outputs.Textbox(label="Text_confidence_score"),254 gr.outputs.Textbox(label="word-by-word").style(show_copy_button=True),255 ],256) 257 258demo.launch()