irulBES/ML-UJI
0
1import re2import joblib3import pandas as pd4import PyPDF25import gradio as gr6from fuzzywuzzy import fuzz7 8# PATH2 = '/content/drive/MyDrive/ML UJI/Model Testing/'9model_path = '600crf (2).pkl'10 11# Function to clean the page12def clean_page(text):13 text = text.replace("Mahkamah Agung Republik Indonesia\nMahkamah Agung Republik Indonesia\nMahkamah Agung Republik Indonesia\nMahkamah Agung Republik Indonesia\nMahkamah Agung Republik Indonesia\nDirektori Putusan Mahkamah Agung Republik Indonesia\nputusan.mahkamahagung.go.id\n", "")14 text = text.replace("\nDisclaimer\nKepaniteraan Mahkamah Agung Republik Indonesia berusaha untuk selalu mencantumkan informasi paling kini dan akurat sebagai bentuk komitmen Mahkamah Agung untuk pelayanan publik, transparansi dan akuntabilitas\npelaksanaan fungsi peradilan. Namun dalam hal-hal tertentu masih dimungkinkan terjadi permasalahan teknis terkait dengan akurasi dan keterkinian informasi yang kami sajikan, hal mana akan terus kami perbaiki dari waktu kewaktu.\nDalam hal Anda menemukan inakurasi informasi yang termuat pada situs ini atau informasi yang seharusnya ada, namun belum tersedia, maka harap segera hubungi Kepaniteraan Mahkamah Agung RI melalui :\nEmail : kepaniteraan@mahkamahagung.go.id", "")15 text = text.replace("Telp : 021-384 3348 (ext.318)", "")16 text = re.sub(r'\nHalaman \d+ dari \d+ .*', '', text)17 text = re.sub(r'Halaman \d+ dari \d+ .*', '', text)18 text = re.sub(r'\nHal. \d+ dari \d+ .*', '', text)19 text = re.sub(r'Hal. \d+ dari \d+ .*', '', text)20 return text.strip()21 22# Function to read and clean text from PDF23def read_pdf(file_pdf):24 try:25 pdf_text = ''26 pdf_file = open(file_pdf, 'rb')27 pdf_reader = PyPDF2.PdfReader(pdf_file)28 29 for page_num in range(len(pdf_reader.pages)):30 page = pdf_reader.pages[page_num]31 text = clean_page(page.extract_text())32 pdf_text += ' ' + text33 34 pdf_file.close()35 return pdf_text.strip()36 37 except Exception as e:38 print("Error:", e)39 40# Function to clean the text41def clean_text(text):42 text = text.replace('P U T U S A N', 'PUTUSAN').replace('T erdakwa', 'Terdakwa').replace('T empat', 'Tempat').replace('T ahun', 'Tahun')43 text = text.replace('P E N E T A P A N', 'PENETAPAN').replace('J u m l a h', 'Jumlah').replace('M E N G A D I L I', 'MENGADILI')44 text = re.sub(r'Halaman \d+', ' ', text)45 text = text.replace('\uf0d8', '').replace('\uf0b7', '').replace('\n', ' ')46 text = re.sub(r'([“”"])', r' \1 ', text)47 text = re.sub(r'\s+', ' ', text)48 return text.strip()49 50# Function for CRF representation51def representation_crf(text):52 content_results = {"doc": [], "fragment": [], "token": []}53 54 fragments = text.split(';')55 56 for fragment_idx, fragment in enumerate(fragments):57 fragment_str = f"fragment:{fragment_idx}"58 doc_str = f"doc:1"59 60 fragment = re.sub(r'([\/,\.():;])', r' \1 ', fragment)61 tokens = re.findall(r'\S+', fragment)62 63 for token in tokens:64 content_results["doc"].append(doc_str)65 content_results["fragment"].append(fragment_str)66 content_results["token"].append(token)67 68 new_data = pd.DataFrame(content_results)69 70 return new_data71 72# Fragment getter class73class FragmentGetter(object):74 75 def __init__(self, data):76 self.n_frag = 177 self.data = data78 self.empty = False79 agg_func = lambda s: [(a) for (a) in zip(s['token'].values.tolist())]80 self.grouped = self.data.groupby('fragment').apply(agg_func)81 self.fragments = [f for f in self.grouped]82 83 def get_next(self):84 try:85 f = self.grouped['Fragment: {}'.format(self.n_frag)]86 self.n_frag += 187 return f88 except:89 return None90 91# Function to convert token to features92def token2features(frag, i):93 token = frag[i][0]94 95 features = {96 'bias': 1.0,97 'token': token98 }99 100 # Features for previous token101 if i > 0:102 features.update({103 'prev1': frag[i - 1][0]104 105 })106 else:107 features['BOF'] = True # Beginning of fragment108 109 if i > 1:110 features.update({111 'prev2': frag[i - 2][0]112 })113 114 # Features for next token115 if i < len(frag) - 1:116 features.update({117 'next1': frag[i + 1][0]118 })119 else:120 features['EOF'] = True # End of fragment121 122 if i < len(frag) - 2:123 features.update({124 'next2': frag[i + 2][0]125 })126 127 return features128 129# Function to convert fragment to features130def frag2features(frag):131 return [token2features(frag, i) for i in range(len(frag))]132 133def frag2labels(frag):134 return [label for token, label in frag]135 136# Function to process a single sentence and return the resulting data137def process_sentence(sentence):138 data = representation_crf(sentence)139 140 # Feature engineering141 X_crf = data.drop(['doc'], axis=1)142 getter = FragmentGetter(X_crf)143 fragments = getter.fragments144 X = [frag2features(f) for f in fragments]145 146 # Predict147 crf = joblib.load(model_path)148 y_pred = crf.predict(X)149 150 # Input label to dataset151 flat_predictions = [tag for sentence_tags in y_pred for tag in sentence_tags]152 assert len(flat_predictions) == len(data['token'])153 data['label'] = flat_predictions154 155 return data156 157# Function to check entities and collect them158# Function to check entities and collect them159def check_entity(df):160 tokens = df['token'].tolist()161 labels = df['label'].tolist()162 163 entities = {}164 current_entity = ''165 current_label = ''166 167 for token, label in zip(tokens, labels):168 if label == 'O':169 if current_entity:170 # Simpan entitas yang selesai jika label saat ini adalah O171 if current_label not in entities:172 entities[current_label] = set()173 entities[current_label].add(current_entity.strip())174 current_entity = ''175 current_label = ''176 continue177 178 entity_type = label.split('_')[-1]179 180 if label.startswith('B_'):181 # Simpan entitas yang selesai jika ada182 if current_entity:183 if current_label not in entities:184 entities[current_label] = set()185 entities[current_label].add(current_entity.strip())186 187 # Mulai entitas baru188 current_entity = token189 current_label = entity_type190 191 elif label.startswith('I_') and current_label == entity_type:192 current_entity += ' ' + token193 194 elif label.startswith('E_') and current_label == entity_type:195 current_entity += ' ' + token196 # Simpan entitas yang lengkap197 if current_label not in entities:198 entities[current_label] = set()199 entities[current_label].add(current_entity.strip())200 current_entity = ''201 current_label = ''202 203 # Tambahkan entitas terakhir jika ada204 if current_entity:205 if current_label not in entities:206 entities[current_label] = set()207 entities[current_label].add(current_entity.strip())208 209 return entities210 211 212 213# Function to format the entities for display214def format_entities(entities):215 key_mapping = {216 "VERN": "Nomor Putusan",217 "DEFN": "Terdakwa",218 "CRIA": "Tindak pidana",219 "PENA": "Tuntutan Hukuman",220 "ARTV": "Pasal yang Dilanggar",221 "PUNI": "Putusan Hukuman",222 "JUDP": "Hakim Ketua",223 "JUDG": "Hakim Anggota",224 "TIMV": "Tanggal Perkara",225 "REGI": "Panitera",226 "PROS": "Penuntut Umum"227 }228 229 def remove_similar(items, threshold=80):230 unique_items = []231 for item in items:232 is_covered = False233 for other_item in unique_items:234 if fuzz.partial_ratio(item, other_item) > threshold:235 is_covered = True236 # Keep the longer item if similar237 if len(item) > len(other_item):238 unique_items.remove(other_item)239 unique_items.append(item)240 break241 if not is_covered:242 unique_items.append(item)243 return unique_items244 245 for entity in entities:246 entities[entity] = remove_similar(entities[entity])247 248 formatted_entities = []249 for key, value in entities.items():250 deskripsi = key_mapping.get(key, key)251 formatted_entities.append(f"{deskripsi}: {' | '.join(value)}")252 return '\n'.join(formatted_entities)253 254# Function to visualize tokens with labels using HighlightedText255# Function to visualize tokens with labels using HighlightedText256def visualize_ner(df):257 tokens = df['token'].tolist()258 labels = df['label'].tolist()259 260 entities = []261 current_entity = ""262 current_label = ""263 264 for token, label in zip(tokens, labels):265 if label == "O":266 if current_entity:267 entities.append((current_entity, current_label))268 current_entity = ""269 current_label = ""270 entities.append((token, None))271 else:272 entity_type = label.split('_')[-1]273 if label.startswith('B_'):274 # Simpan entitas yang sedang berlangsung275 if current_entity:276 entities.append((current_entity, current_label))277 current_entity = token278 current_label = entity_type279 elif label.startswith('I_') and current_label == entity_type:280 current_entity += " " + token281 elif label.startswith('E_') and current_label == entity_type:282 current_entity += " " + token283 entities.append((current_entity, current_label))284 current_entity = ""285 current_label = ""286 287 # Tambahkan entitas terakhir jika ada288 if current_entity:289 entities.append((current_entity, current_label))290 291 return entities292 293 294# Function to process the input text295def process_text(input_text):296 pecahan = input_text.split(';')297 298 results = []299 all_entities = {}300 visualizations = []301 302 for kalimat in pecahan:303 result = process_sentence(kalimat)304 results.append(result)305 306 # Extract and collect entities307 entities = check_entity(result)308 for entity_type, entity_set in entities.items():309 if entity_type not in all_entities:310 all_entities[entity_type] = set()311 all_entities[entity_type].update(entity_set)312 313 # Generate visualization314 visualizations.extend(visualize_ner(result))315 316 # Combine all results into a single DataFrame317 df = pd.concat(results, ignore_index=True)318 319 # Convert the entities dictionary to a list of tuples and remove duplicates320 final_entities = {entity_type: list(entity_set) for entity_type, entity_set in all_entities.items()}321 322 return format_entities(final_entities), visualizations323 324# Function to handle uploaded PDF325def process_pdf(pdf_file):326 text = read_pdf(pdf_file.name)327 return process_text(clean_text(text))328 329# Define Gradio interface using Blocks330with gr.Blocks() as iface:331 gr.Markdown("## Entity Extraction")332 gr.Markdown("Enter your text to extract entities or upload a PDF file.")333 334 with gr.Tab("Input Text"):335 text_input = gr.Textbox(label="Input Text")336 text_button = gr.Button("Extract Entities")337 text_output_entities = gr.Textbox(label="Extracted Entities", lines=10)338 text_output_visualization = gr.HighlightedText(label="Visualization")339 text_button.click(fn=process_text, inputs=text_input, outputs=[text_output_entities, text_output_visualization])340 341 with gr.Tab("Upload PDF"):342 pdf_input = gr.File(label="Upload PDF")343 pdf_button = gr.Button("Extract Entities")344 pdf_output_entities = gr.Textbox(label="Extracted Entities", lines=10)345 pdf_output_visualization = gr.HighlightedText(label="Visualization")346 pdf_button.click(fn=process_pdf, inputs=pdf_input, outputs=[pdf_output_entities, pdf_output_visualization])347 348# Launch Gradio app349iface.launch()