IAMJB/RadEvalModernBERT
519k
1---2library_name: transformers3license: mit4base_model:5- answerdotai/ModernBERT-base6---7**Inference**:8 9```python10import torch11from transformers import AutoTokenizer, AutoModel12import torch.nn.functional as F13 14# Pick one sentence15sentence = "The patient has a right pneumothorax."16 17# Load pretrained model and tokenizer18model_name = "IAMJB/RadEvalModernBERT"19tokenizer = AutoTokenizer.from_pretrained(model_name)20model = AutoModel.from_pretrained(model_name)21 22# Put model in eval mode and set device23device = torch.device("cuda" if torch.cuda.is_available() else "cpu")24model.to(device)25model.eval()26 27# Tokenize input28inputs = tokenizer(sentence, return_tensors="pt", truncation=True, padding=True).to(device)29 30# Get embeddings31with torch.no_grad():32 outputs = model(**inputs, output_hidden_states=True)33 last_hidden_state = outputs.hidden_states[-1]34 cls_embedding = last_hidden_state[:, 0, :] # CLS token35 cls_embedding = F.normalize(cls_embedding, p=2, dim=1)36 37 38print("Sentence:", sentence)39print("Embedding shape:", cls_embedding.shape)40```41 42 43 44### Similarity heatmap example45 46 47```python48import argparse49import numpy as np50import matplotlib.pyplot as plt51import torch52import seaborn as sns53from transformers import AutoTokenizer, AutoModel54 55def get_cls_embeddings(model, tokenizer, texts, device):56 """Get CLS token embeddings for a list of texts."""57 embeddings = []58 59 for text in texts:60 # Tokenize the text61 inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)62 inputs = {k: v.to(device) for k, v in inputs.items()}63 64 # Get the embeddings (use CLS token)65 with torch.no_grad():66 outputs = model(**inputs, output_hidden_states=True)67 # Use the last hidden state68 last_hidden_state = outputs.hidden_states[-1]69 # Extract CLS token (first token) embedding70 cls_embedding = last_hidden_state[:, 0, :]71 embeddings.append(cls_embedding.cpu().numpy()[0])72 73 return np.array(embeddings)74 75def compute_similarities(embeddings):76 """Compute cosine similarity between embeddings."""77 # Normalize embeddings78 normalized_embeddings = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)79 # Compute similarity matrix80 similarity_matrix = np.matmul(normalized_embeddings, normalized_embeddings.T)81 return similarity_matrix82 83def plot_heatmap(similarity_matrix, labels, output_path="cls_embedding_similarities.png"):84 """Generate a heatmap visualization of the similarity matrix."""85 plt.figure(figsize=(10, 8))86 87 # Find min value to set as vmin (or use 0.6 as a reasonable value)88 min_val = max(0.0, np.min(similarity_matrix))89 90 # Create the heatmap with adjusted color scale91 ax = sns.heatmap(92 similarity_matrix,93 annot=True,94 fmt=".3f",95 cmap="viridis", # Better colormap for distinguishing high values96 vmin=min_val, # Start from minimum value or 0.697 vmax=1.0,98 xticklabels=labels,99 yticklabels=labels,100 cbar_kws={"label": "Similarity"}101 )102 103 # Add title and adjust layout104 plt.title("CLS Token Embedding Similarities")105 plt.tight_layout()106 107 # Rotate x-axis labels for better readability108 plt.xticks(rotation=90)109 110 # Save the figure111 plt.savefig(output_path, dpi=300, bbox_inches="tight")112 print(f"Heatmap saved to {output_path}")113 114 # Show the plot115 plt.show()116 117def main():118 # Medical terms to compare119 medical_terms = [120 "large right pneumothorax",121 "right pneumothorax",122 "pneumonia in the right lower lobe",123 "consolidation in the right lower lobe",124 "right 9th rib fracture",125 "left 9th rib fracture",126 "left 5th rib fracture",127 "5th metatarsal fracture",128 "no pneumothorax is present",129 "prior consolidation has cleared",130 "no rib fractures"131 ]132 133 # Set the device134 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")135 print(f"Using device: {device}")136 137 # Load the tokenizer138 tokenizer = AutoTokenizer.from_pretrained(IAMJB/RadEvalModernBERT)139 140 # Load the model141 model = AutoModel.from_pretrained(IAMJB/RadEvalModernBERT)142 model.to(device)143 model.eval()144 145 # Get CLS token embeddings for the medical terms146 print("Generating CLS token embeddings...")147 embeddings = get_cls_embeddings(model, tokenizer, medical_terms, device)148 149 # Compute similarities150 print("Computing similarity matrix...")151 similarity_matrix = compute_similarities(embeddings)152 153 # Plot and save the heatmap154 print("Generating heatmap...")155 plot_heatmap(similarity_matrix, medical_terms, "cls_embedding_similarities.png")156 157 print("Done!")158 159if __name__ == "__main__":160 main()161```162 163164 165 166**Reference**:167```168@inproceedings{xu-etal-2025-radeval,169 title = "{R}ad{E}val: A framework for radiology text evaluation",170 author = "Xu, Justin and171 Zhang, Xi and172 Abderezaei, Javid and173 Bauml, Julie and174 Boodoo, Roger and175 Haghighi, Fatemeh and176 Ganjizadeh, Ali and177 Brattain, Eric and178 Van Veen, Dave and179 Meng, Zaiqiao and180 Eyre, David W and181 Delbrouck, Jean-Benoit",182 editor = {Habernal, Ivan and183 Schulam, Peter and184 Tiedemann, J{\"o}rg},185 booktitle = "Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",186 month = nov,187 year = "2025",188 address = "Suzhou, China",189 publisher = "Association for Computational Linguistics",190 url = "https://aclanthology.org/2025.emnlp-demos.40/",191 doi = "10.18653/v1/2025.emnlp-demos.40",192 pages = "546--557",193 ISBN = "979-8-89176-334-0",194}195```