johnodonovan/GPUtil
1
1"""2GPU and Model specifications for CloudPhysician ICU Monitoring System3Contains inference benchmarks and hardware specifications.4"""5 6# GPU/CPU Specifications7# VRAM/Memory in GB, TFLOPs (FP16), estimated retail price USD8GPU_SPECS = {9 "RTX 4080": {10 "vram_gb": 16,11 "tflops_fp16": 48.7,12 "tflops_fp32": 48.7,13 "tensor_tflops": 780,14 "price_usd": 1199, # MSRP15 "tdp_watts": 320,16 "architecture": "Ada Lovelace",17 "is_cpu": False,18 },19 "RTX 5080": {20 "vram_gb": 16,21 "tflops_fp16": 56.2,22 "tflops_fp32": 56.2,23 "tensor_tflops": 900,24 "price_usd": 999, # MSRP25 "tdp_watts": 360,26 "architecture": "Blackwell",27 "is_cpu": False,28 },29 "RTX 5090": {30 "vram_gb": 32,31 "tflops_fp16": 104.8,32 "tflops_fp32": 104.8,33 "tensor_tflops": 1680,34 "price_usd": 1999, # MSRP35 "tdp_watts": 575,36 "architecture": "Blackwell",37 "is_cpu": False,38 },39 "NVIDIA L40S": {40 "vram_gb": 48,41 "tflops_fp16": 91.6,42 "tflops_fp32": 91.6,43 "tensor_tflops": 733,44 "price_usd": 8500, # MSRP (datacenter)45 "tdp_watts": 350,46 "architecture": "Ada Lovelace (Datacenter)",47 "is_cpu": False,48 },49 "Apple M5": {50 "vram_gb": 32, # Unified memory (configurable)51 "tflops_fp16": 14.0,52 "tflops_fp32": 7.0,53 "tensor_tflops": 38, # Neural Engine TOPS54 "price_usd": 2499, # Mac Studio base price estimate55 "tdp_watts": 75,56 "architecture": "Apple Silicon (CPU+GPU+Neural Engine)",57 "is_cpu": True,58 },59 "Intel Core i7-14700K": {60 "vram_gb": 64, # System RAM (typical config)61 "tflops_fp16": 1.2,62 "tflops_fp32": 0.6,63 "tensor_tflops": 0, # No dedicated tensor cores64 "price_usd": 419,65 "tdp_watts": 125,66 "architecture": "Raptor Lake Refresh (CPU)",67 "is_cpu": True,68 },69}70 71# Model inference times in milliseconds at 1024x768 resolution72# These are estimated/benchmarked values per inference73MODEL_INFERENCE_MS = {74 "Mask2Former": {75 "RTX 4080": 42,76 "RTX 5080": 35,77 "RTX 5090": 18,78 "NVIDIA L40S": 22,79 "Apple M5": 65,80 "Intel Core i7-14700K": 270,81 "description": "Transformer-based instance segmentation",82 "use_case": "Patient body segmentation, bed occupancy detection",83 "vram_usage_mb": 3200,84 },85 "YOLOv8": {86 "RTX 4080": 8,87 "RTX 5080": 6,88 "RTX 5090": 3,89 "NVIDIA L40S": 5,90 "Apple M5": 12,91 "Intel Core i7-14700K": 45,92 "description": "Real-time object detection",93 "use_case": "Medical equipment detection, staff tracking",94 "vram_usage_mb": 850,95 },96 "RF-DETR": {97 "RTX 4080": 28,98 "RTX 5080": 22,99 "RTX 5090": 12,100 "NVIDIA L40S": 16,101 "Apple M5": 45,102 "Intel Core i7-14700K": 180,103 "description": "Transformer-based detection with deformable attention",104 "use_case": "High-accuracy patient pose estimation, IV line detection",105 "vram_usage_mb": 2400,106 },107 "RDInfer": {108 "RTX 4080": 4660,109 "RTX 5080": 3880,110 "RTX 5090": 2000,111 "NVIDIA L40S": 2440,112 "Apple M5": 7000,113 "Intel Core i7-14700K": 30000,114 "description": "Classify video clip as RD or not based on 20 second video clip at 8 FPS. Includes time for FFT along H, W & T",115 "use_case": "Detect high work of breathing in patients",116 "vram_usage_mb": 16000,117 },118 "RDInferOnly": {119 "RTX 4080": 466,120 "RTX 5080": 388,121 "RTX 5090": 200,122 "NVIDIA L40S": 244,123 "Apple M5": 700,124 "Intel Core i7-14700K": 3000,125 "description": "Classify video clip as RD or not based on 20 second video clip at 8 FPS. Includes time for FFT along H, W & T",126 "use_case": "Detect high work of breathing in patients",127 "vram_usage_mb": 1600,128 },129 "BedOccupancy_1": {130 "RTX 4080": 16,131 "RTX 5080": 14,132 "RTX 5090": 7,133 "NVIDIA L40S": 9,134 "Apple M5": 25,135 "Intel Core i7-14700K": 105,136 "description": "Classify whether the input image has an empty bed, occupied bed or no bed at all",137 "use_case": "Determine whether a given bed is occupied and bypass running other heavy models like RD when bed is not occupied",138 "vram_usage_mb": 4000,139 },140 "NeonatalRR": {141 "RTX 4080": 93200,142 "RTX 5080": 77600,143 "RTX 5090": 40000,144 "NVIDIA L40S": 48800,145 "Apple M5": 140000,146 "Intel Core i7-14700K": 600000,147 "description": "Determine respiratory rate in bins of 3 breaths per minute of a baby breathing in 720P 24FPS 20 second video clip",148 "use_case": "Counting respiratory is slow and error prone for humans. We automate this",149 "vram_usage_mb": 32000,150 },151 "vsitter": {152 "RTX 4080": 12,153 "RTX 5080": 10,154 "RTX 5090": 5,155 "NVIDIA L40S": 6,156 "Apple M5": 18,157 "Intel Core i7-14700K": 75,158 "description": "EVA-02 vision transformer model. P95 inference time from 20 runs",159 "use_case": "Backbone model to predict the likelihood of a fall",160 "vram_usage_mb": 1341,161 },162 "Netra": {163 "RTX 4080": 373,164 "RTX 5080": 310,165 "RTX 5090": 160,166 "NVIDIA L40S": 195,167 "Apple M5": 560,168 "Intel Core i7-14700K": 2400,169 "description": "Series of models for segmentation, classification and OCR with post-processing to output JSON for cardiac monitor vitals",170 "use_case": "Fetch vitals from cardiac monitor image",171 "vram_usage_mb": 1000,172 },173}174 175# Resolution scaling factors (relative to 1024x768 baseline)176RESOLUTION_PRESETS = {177 "640x480 (VGA)": {"width": 640, "height": 480, "scale_factor": 0.4},178 "800x600 (SVGA)": {"width": 800, "height": 600, "scale_factor": 0.6},179 "1024x768 (XGA)": {"width": 1024, "height": 768, "scale_factor": 1.0},180 "1280x720 (720p HD)": {"width": 1280, "height": 720, "scale_factor": 1.2},181 "1920x1080 (1080p FHD)": {"width": 1920, "height": 1080, "scale_factor": 2.7},182 "2560x1440 (1440p QHD)": {"width": 2560, "height": 1440, "scale_factor": 4.8},183 "3840x2160 (4K UHD)": {"width": 3840, "height": 2160, "scale_factor": 10.8},184}185 186# Default resolution key187DEFAULT_RESOLUTION = "1024x768 (XGA)"188 189 190def get_scaled_inference_time(model: str, gpu: str, resolution: str) -> float:191 """192 Calculate inference time adjusted for resolution.193 Higher resolutions increase inference time proportionally.194 """195 base_time = MODEL_INFERENCE_MS[model][gpu]196 scale = RESOLUTION_PRESETS[resolution]["scale_factor"]197 return base_time * scale198 199 200def calculate_max_fps_per_gpu(models: list, gpu: str, resolution: str) -> float:201 """202 Calculate the maximum FPS a single GPU can handle running all selected models.203 Returns FPS as float.204 """205 total_inference_ms = sum(206 get_scaled_inference_time(model, gpu, resolution) for model in models207 )208 if total_inference_ms <= 0:209 return float("inf")210 return 1000.0 / total_inference_ms211 212 213def calculate_gpus_needed(214 num_beds: int,215 models: list,216 gpu: str,217 resolution: str,218 target_fps: float,219 inference_interval_sec: float = 0,220 include_vram: bool = False,221) -> dict:222 """223 Calculate number of GPUs needed for the workload.224 225 Args:226 num_beds: Number of camera feeds (1 per bed)227 models: List of model names to run228 gpu: GPU type229 resolution: Resolution preset name230 target_fps: Target frames per second231 inference_interval_sec: Seconds between inferences (0 = continuous)232 include_vram: Whether to include VRAM as a limiting factor233 234 Returns:235 Dictionary with calculation details236 """237 # Calculate total inference time per frame for all models238 total_inference_ms = sum(239 get_scaled_inference_time(model, gpu, resolution) for model in models240 )241 242 # Effective FPS requirement243 if inference_interval_sec > 0:244 effective_fps = 1.0 / inference_interval_sec245 else:246 effective_fps = target_fps247 248 # Max streams a single GPU can handle249 max_fps_per_gpu = 1000.0 / total_inference_ms if total_inference_ms > 0 else float("inf")250 streams_per_gpu = max_fps_per_gpu / effective_fps if effective_fps > 0 else float("inf")251 252 # Calculate GPUs needed (based on inference time only)253 if streams_per_gpu >= num_beds:254 gpus_needed = 1255 else:256 gpus_needed = int(num_beds / streams_per_gpu) + (1 if num_beds % streams_per_gpu > 0 else 0)257 258 # VRAM calculations (always compute for display, but only apply if include_vram is True)259 total_vram_needed_mb = sum(MODEL_INFERENCE_MS[m]["vram_usage_mb"] for m in models)260 vram_available_mb = GPU_SPECS[gpu]["vram_gb"] * 1024261 vram_limited_streams = int(vram_available_mb / total_vram_needed_mb) if total_vram_needed_mb > 0 else float("inf")262 263 vram_insufficient = False264 vram_limited = False265 266 # Only apply VRAM constraints if include_vram is True267 if include_vram:268 # Handle case where VRAM is insufficient for even one stream269 vram_insufficient = vram_limited_streams < 1270 if vram_insufficient:271 # Each GPU can only run one partial stream (models won't fit together)272 # User needs one GPU per bed in this case273 streams_per_gpu = 1274 gpus_needed = num_beds275 elif vram_limited_streams < streams_per_gpu:276 # Adjust for VRAM limitations277 vram_limited = True278 streams_per_gpu = vram_limited_streams279 gpus_needed = max(gpus_needed, int(num_beds / streams_per_gpu) + (1 if num_beds % streams_per_gpu > 0 else 0))280 281 return {282 "gpus_needed": max(1, gpus_needed),283 "inference_time_ms": total_inference_ms,284 "max_fps_per_gpu": max_fps_per_gpu,285 "streams_per_gpu": streams_per_gpu,286 "effective_fps": effective_fps,287 "vram_per_stream_mb": total_vram_needed_mb,288 "vram_available_mb": vram_available_mb,289 "vram_limited": vram_limited,290 "vram_insufficient": vram_insufficient,291 "include_vram": include_vram,292 "total_cost": max(1, gpus_needed) * GPU_SPECS[gpu]["price_usd"],293 "total_power_watts": max(1, gpus_needed) * GPU_SPECS[gpu]["tdp_watts"],294 }295 296 297