VincentGOURBIN/MeetingNotes-Voxtral-Analysis

1"""2Token usage tracking utility for MeetingNotes HF Spaces.3 4This module provides a centralized way to track and report token consumption5for Transformers-based processing in HF Spaces environment.6"""7 8 9class TokenTracker:10    """11    Centralized token usage tracking for HF Spaces.12    13    Tracks input and output tokens across different chunks and processing modes14    to provide comprehensive usage statistics.15    """16    17    def __init__(self, mode: str = "Transformers-8bit"):18        self.mode = mode19        self.reset()20    21    def reset(self):22        """Reset all counters."""23        self.chunks_processed = 024        self.total_input_tokens = 025        self.total_output_tokens = 026        self.synthesis_input_tokens = 027        self.synthesis_output_tokens = 028    29    def set_mode(self, mode: str):30        """Set the processing mode for reporting."""31        self.mode = mode32    33    def add_chunk_tokens(self, input_tokens: int, output_tokens: int):34        """Add tokens from a chunk processing."""35        self.chunks_processed += 136        self.total_input_tokens += input_tokens37        self.total_output_tokens += output_tokens38        39        print(f"📊 Stats {self.mode} Chunk {self.chunks_processed} - Input: {input_tokens} tokens, Output: {output_tokens} tokens")40    41    def add_synthesis_tokens(self, input_tokens: int, output_tokens: int):42        """Add tokens from synthesis processing."""43        self.synthesis_input_tokens = input_tokens44        self.synthesis_output_tokens = output_tokens45        46        print(f"📊 Stats {self.mode} Synthesis - Input: {input_tokens} tokens, Output: {output_tokens} tokens")47    48    def print_summary(self):49        """Print final token usage summary."""50        total_input = self.total_input_tokens + self.synthesis_input_tokens51        total_output = self.total_output_tokens + self.synthesis_output_tokens52        grand_total = total_input + total_output53        54        print(f"\n📊 === TOKEN USAGE SUMMARY ({self.mode}) ===")55        print(f"📦 Chunks processed: {self.chunks_processed}")56        print(f"📥 Total input tokens: {total_input:,}")57        print(f"📤 Total output tokens: {total_output:,}")58        print(f"🔢 Grand total: {grand_total:,} tokens")59        60        if self.synthesis_input_tokens > 0:61            print(f"   • Chunk analysis: {self.total_input_tokens + self.total_output_tokens:,} tokens")62            print(f"   • Final synthesis: {self.synthesis_input_tokens + self.synthesis_output_tokens:,} tokens")63        64        print("=" * 50)