mr-kush/urgency-classifier-retraining
0
1#model_piepline.py2 3import os4import json5from datetime import datetime, UTC, timezone, timedelta6from huggingface_hub import HfApi7from huggingface_hub.utils import HfHubHTTPError8import numpy as np9import torch10import torch.nn as nn11import torch.nn.functional as F12import wandb13from wandb import AlertLevel14import matplotlib.pyplot as plt15import seaborn as sns16import requests17 18from transformers import (19 AutoTokenizer, DataCollatorWithPadding,20 AutoModelForSequenceClassification, TrainingArguments,21 Trainer, EarlyStoppingCallback22)23from sklearn.metrics import (24 accuracy_score, f1_score,25 precision_score, recall_score,26 classification_report, confusion_matrix27)28 29 30 31# function to tokenize a batch of examples32def tokenize_function(examples, tokenizer, text_column: str):33 """Helper function for tokenization (pickle-safe for HF caching)."""34 return tokenizer(examples[text_column], truncation=True)35 36def sanitize_training_args(training_args):37 """Convert TrainingArguments to JSON-serializable dictionary."""38 if not training_args:39 return {}40 args_dict = training_args.to_dict()41 clean_dict = {}42 for k, v in args_dict.items():43 try:44 json.dumps({k: v})45 clean_dict[k] = v46 except TypeError:47 clean_dict[k] = str(v) # fallback: convert to string48 return clean_dict49 50 51 52 53 54class FocalLossMultiClass(nn.Module):55 """Implementation of Focal Loss for multi-class classification."""56 57 def __init__(self, gamma: float = 2.0, alpha: float = 0.25, reduction: str = 'mean'):58 """59 Args:60 gamma (float): Focusing parameter. Default=2.061 alpha (float): Weighting factor for class imbalance. Default=0.2562 reduction (str): 'mean', 'sum', or 'none'. Default='mean'63 """64 super().__init__()65 self.gamma = gamma66 self.alpha = alpha67 self.reduction = reduction68 69 def forward(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:70 ce_loss = F.cross_entropy(logits, targets, reduction='none')71 pt = torch.exp(-ce_loss)72 focal_loss = self.alpha * (1 - pt) ** self.gamma * ce_loss73 74 if self.reduction == 'mean':75 return focal_loss.mean()76 elif self.reduction == 'sum':77 return focal_loss.sum()78 return focal_loss79 80 81class FocalLossTrainer(Trainer):82 """Custom Hugging Face Trainer using Focal Loss."""83 84 def __init__(self, class_weights: torch.Tensor = None, *args, **kwargs):85 """86 Args:87 class_weights (torch.Tensor, optional): Tensor for weighting classes in loss.88 """89 super().__init__(*args, **kwargs)90 # self.class_weights = class_weights.to(self.model.device) # optional91 92 def compute_loss(self, model: nn.Module, inputs: dict, return_outputs: bool = False, **kwargs) -> torch.Tensor:93 labels = inputs.get("labels")94 outputs = model(**inputs)95 logits = outputs.get("logits")96 loss_fct = FocalLossMultiClass()97 loss = loss_fct(logits, labels)98 return (loss, outputs) if return_outputs else loss99 100 101class GrievanceClassifier:102 """Grievance classification model wrapper with training, evaluation, and HF Hub integration."""103 104 def __init__(105 self,106 model_checkpoint: str,107 num_labels: int,108 id2label: dict,109 label2id: dict, 110 hf_token: str,111 wandb_api_key: str,112 wandb_project_name: str, 113 ):114 """115 Args:116 hf_token(str): HF-token for HF Hub Write Acess117 model_checkpoint (str): HF model checkpoint, e.g., 'xlm-roberta-base'118 num_labels (int): Number of classes for classification119 id2label (dict): Mapping from label IDs to string labels120 label2id (dict): Mapping from string labels to label IDs121 wandb_api_key (str) : WandB Access API key 122 wandb_project_name (str): WandB project name for experiment tracking123 """124 self.model_checkpoint = model_checkpoint125 self.num_labels = num_labels126 self.id2label = id2label127 self.label2id = label2id128 self.hf_token = hf_token129 self.api = HfApi()130 131 # Login wandb132 wandb.login(key=wandb_api_key)133 self.wandb_project_name = wandb_project_name134 135 136 137 138 139 # Load tokenizer and model140 self.tokenizer = AutoTokenizer.from_pretrained(model_checkpoint,141 use_fast=True, 142 token= self.hf_token)143 self.model = AutoModelForSequenceClassification.from_pretrained(144 model_checkpoint,145 num_labels=num_labels,146 id2label=id2label,147 label2id=label2id, 148 token= self.hf_token149 )150 151 def tokenize_dataset(self, dataset, text_column: str = "grievance", remove_columns: bool = True, batched: bool = True):152 """153 Tokenize a HF Dataset or DatasetDict using the class tokenizer.154 155 Args:156 dataset: HF Dataset or DatasetDict to tokenize157 text_column (str): Name of the column containing the text. Default="grievance"158 remove_columns (bool): Whether to remove the original text column after tokenization. Default=True159 batched (bool): Whether to batch examples during tokenization. Default=True160 161 Returns:162 tokenized_dataset: Tokenized HF Dataset or DatasetDict163 """164 165 166 tokenized_dataset = dataset.map(167 lambda examples: tokenize_function(examples, self.tokenizer, text_column),168 batched=batched169 )170 171 if remove_columns and text_column in tokenized_dataset.column_names:172 tokenized_dataset = tokenized_dataset.remove_columns([text_column])173 174 return tokenized_dataset175 176 @staticmethod177 def compute_metrics(eval_pred: tuple) -> dict:178 """179 Compute classification metrics.180 181 Args:182 eval_pred (tuple): (logits, labels) from trainer.predict183 184 Returns:185 dict: Accuracy, F1 (macro & weighted), precision, recall186 """187 logits, labels = eval_pred188 predictions = np.argmax(logits, axis=-1)189 return {190 "accuracy": accuracy_score(labels, predictions),191 "f1_macro": f1_score(labels, predictions, average="macro", zero_division=0),192 "f1_weighted": f1_score(labels, predictions, average="weighted", zero_division=0),193 "precision_macro": precision_score(labels, predictions, average="macro", zero_division=0),194 "recall_macro": recall_score(labels, predictions, average="macro", zero_division=0),195 "precision_weighted": precision_score(labels, predictions, average="weighted", zero_division=0),196 "recall_weighted": recall_score(labels, predictions, average="weighted", zero_division=0)197 }198 199 def train(200 self,201 train_dataset,202 eval_dataset,203 output_dir: str | None = None,204 hf_training_args: dict | None = None,205 early_stopping_patience: int = 2,206 early_stopping_threshold: float=0.001207 ):208 """209 Train the model using HF Trainer with Focal Loss.210 211 Args:212 train_dataset: HF Dataset or DatasetDict for training213 eval_dataset: HF Dataset or DatasetDict for validation214 wandb_project_name (str): WandB project name for experiment tracking215 output_dir (str, optional): Directory to save checkpoints216 hf_training_args (dict, optional): Dictionary of HuggingFace TrainingArguments to override defaults217 early_stopping_patience (int): Patience for early stopping218 """219 # Early stopping callback220 early_stopping_callback = EarlyStoppingCallback(221 early_stopping_patience=early_stopping_patience,222 early_stopping_threshold=early_stopping_threshold223 )224 225 226 227 # Tokenize datasets228 train_dataset = self.tokenize_dataset(train_dataset)229 eval_dataset = self.tokenize_dataset(eval_dataset)230 231 # Default training arguments with no logging and no step-wise saving232 self.default_args = {233 "num_train_epochs": 3,234 "per_device_train_batch_size": 16,235 "per_device_eval_batch_size": 32,236 "learning_rate": 2e-5,237 "weight_decay": 0.01,238 "eval_strategy": "steps",239 "eval_steps": 50,240 "logging_steps": 50,241 "load_best_model_at_end": True,242 "metric_for_best_model": "f1_macro",243 "greater_is_better": True,244 "fp16": True,245 "push_to_hub": False,246 "hub_model_id": None,247 "report_to": ["wandb"],248 "logging_dir": "./logs",249 "gradient_accumulation_steps": 1250 }251 252 253 254 # Merge user-provided overrides255 if hf_training_args:256 self.default_args.update(hf_training_args)257 # initalizing hub id258 self.hub_model_id= self.default_args.get('hub_model_id')259 260 # Initialize TrainingArguments261 self.training_args = TrainingArguments(**self.default_args)262 263 # sanitize training arg for metadata file:264 self.sanitize_training_args = (sanitize_training_args(getattr(self, 265 "training_args",266 None))267 if hasattr(self, "training_args")268 else {}269 )270 271 272 # Initialize trainer273 self.trainer = FocalLossTrainer(274 model=self.model,275 args=self.training_args,276 train_dataset=train_dataset,277 eval_dataset=eval_dataset,278 compute_metrics=self.compute_metrics,279 callbacks=[early_stopping_callback],280 data_collator=DataCollatorWithPadding(tokenizer=self.tokenizer),281 processing_class=self.tokenizer282 )283 284 # Start training285 self.trainer.train()286 287 def log_wandb_eval_metrics(288 self,289 y_true,290 y_pred,291 classification_report_dict,292 confusion_matrix_array,293 label_names,294 prefix="final_eval"295 ):296 """Logs classification metrics and confusion matrix to Weights & Biases."""297 try:298 # 1️ Create classification report as W&B Table299 # Prepare rows300 rows = [301 [label,302 round(metrics["precision"], 4),303 round(metrics["recall"], 4),304 round(metrics["f1-score"], 4),305 int(metrics["support"])]306 for label, metrics in classification_report_dict.items()307 if isinstance(metrics, dict) and all(k in metrics for k in ["precision", "recall", "f1-score", "support"])308 ]309 310 # Create table using columns + data argument311 table = wandb.Table(columns=["Class", "Precision", "Recall", "F1-score", "Support"], data=rows)312 313 # 2️ Plot confusion matrix314 fig, ax = plt.subplots(figsize=(6, 6))315 sns.heatmap(confusion_matrix_array, annot=True, fmt="d", cmap="Blues",316 xticklabels=label_names, yticklabels=label_names, ax=ax)317 ax.set_xlabel("Predicted Label")318 ax.set_ylabel("True Label")319 ax.set_title("Confusion Matrix")320 plt.tight_layout()321 cm_image = wandb.Image(fig)322 plt.close(fig)323 324 # 3️ Log both to W&B325 wandb.log({326 f"{prefix}/classification_report_table": table, 327 f"{prefix}/confusion_matrix": cm_image328 }, commit=True)329 330 except Exception as e:331 print(f"[W&B Logging Error] {type(e).__name__}: {e}", flush=True)332 333 334 def _query_deployed_model(self, 335 texts: list[str],336 api_endpoint: str,337 timeout: int = 8) -> list[int]:338 """339 Query a deployed model API and return predicted label IDs.340 341 Args:342 texts (list[str]): List of raw text inputs.343 api_endpoint (str): POST /predict endpoint URL.344 timeout (int): Request timeout in seconds.345 346 Returns:347 List[int]: Predicted label IDs (-1 if prediction failed or unknown).348 """349 350 pred_ids = []351 for txt in texts:352 try:353 resp = requests.post(api_endpoint, json={"text": txt}, timeout=timeout)354 if resp.status_code == 200:355 data = resp.json()356 label_str = data.get("label")357 # Map string label to ID358 pred_id = self.label2id.get(label_str, None)359 if pred_id is None:360 try:361 pred_id = int(label_str)362 except Exception:363 pred_id = -1364 pred_ids.append(pred_id if pred_id is not None else -1)365 else:366 pred_ids.append(-1)367 except Exception:368 pred_ids.append(-1)369 370 return pred_ids371 372 373 def evaluate(374 self,375 test_dataset,376 api_endpoint: str | None = None,377 threshold: float = 0.00,378 deployed_sample_size: int = 300379 ):380 """381 Pure evaluation function: tokenizes test data, predicts labels, computes metrics,382 optionally compares against deployed model, and returns outcomes.383 384 Args:385 test_dataset: Hugging Face Dataset for testing.386 api_endpoint (str, optional): Deployed model /predict API endpoint.387 threshold (float): Minimum F1 macro improvement over deployed model for decision.388 deployed_sample_size (int): Number of samples to query deployed model for F1 comparison.389 390 Returns:391 dict: {392 "predictions": np.ndarray,393 "y_true": np.ndarray,394 "confusion_matrix": np.ndarray,395 "classification_report": dict,396 "f1_macro": float,397 "deployed_f1_macro": float | None,398 "decision": "accepted" | "rejected"399 }400 """401 402 # 1️ Tokenize test dataset403 test_dataset_tokenized = self.tokenize_dataset(test_dataset)404 405 # 2️ Run model predictions406 predictions = self.trainer.predict(test_dataset_tokenized)407 y_true = predictions.label_ids408 y_pred = np.argmax(predictions.predictions, axis=-1)409 410 # 3️ Compute classification report and confusion matrix411 classification_report_dict = classification_report(412 y_true,413 y_pred,414 target_names=list(self.id2label.values()),415 output_dict=True416 )417 labels = list(self.id2label.keys())418 cm = confusion_matrix(y_true, y_pred, labels=labels)419 420 # 4️ Compute current model F1 macro421 current_trained_f1_macro = f1_score(y_true, y_pred, average="macro", zero_division=0)422 423 # 5️ Optionally compare with deployed model F1424 deployed_f1_macro = None425 if api_endpoint:426 raw_test = test_dataset.shuffle(seed=42)427 n = min(deployed_sample_size, len(raw_test))428 texts = raw_test["grievance"][:n]429 true_labels = raw_test["label"][:n] if "label" in raw_test.column_names else raw_test["labels"][:n]430 431 deployed_preds_ids = self._query_deployed_model(texts, api_endpoint)432 433 # Filter out failed predictions (-1)434 paired_true, paired_pred = [], []435 for t, p in zip(true_labels, deployed_preds_ids):436 if p != -1:437 paired_true.append(int(t))438 paired_pred.append(int(p))439 440 if paired_true:441 deployed_f1_macro = f1_score(paired_true, paired_pred, average="macro", zero_division=0)442 else:443 deployed_f1_macro = 0.0444 445 # 6️ Decision logic446 deployed_f1_to_compare = deployed_f1_macro if deployed_f1_macro is not None else 0.0447 decision = "accepted" if current_trained_f1_macro > deployed_f1_to_compare + threshold else "rejected"448 449 # 7️ Return all evaluation outcomes450 return {451 "predictions": y_pred,452 "y_true": y_true,453 "confusion_matrix": cm,454 "classification_report": classification_report_dict,455 "current_trained_f1_macro": current_trained_f1_macro,456 "deployed_f1_macro": deployed_f1_macro,457 "decision": decision458 }459 460 def push_model_to_hub(461 self,462 hub_model_id: str | None = None,463 use_trainer: bool = False,464 commit_message: str = "Push model and tokenizer to Hugging Face Hub",465 ):466 """467 Push the model and tokenizer (or trainer) to the Hugging Face Hub with proper468 version tagging, metadata logging, and safe cleanup.469 470 Args:471 hub_model_id (str): Repository ID on Hugging Face Hub.472 use_trainer (bool): Whether to use trainer.push_to_hub().473 commit_message (str): Custom commit message.474 """475 # version tag476 timestamp= datetime.now(UTC).strftime("%Y%m%d_%H%M%S")477 self.version_tag = f"v{timestamp}"478 479 if hub_model_id is None:480 hub_model_id = getattr(self.training_args, "hub_model_id", None)481 if hub_model_id is None:482 raise ValueError("You must provide a hub_model_id or define it in TrainingArguments.")483 484 self.commit_message = f"{commit_message} ({self.version_tag})"485 metadata_path = "model_metadata.json"486 487 488 try:489 print("Starting model push to Hugging Face Hub...", flush=True)490 491 # Step 1: Push model and tokenizer (or trainer)492 if use_trainer and hasattr(self, "trainer") and self.trainer is not None:493 self.trainer.push_to_hub(commit_message=self.commit_message, token=self.hf_token)494 else:495 self.model.push_to_hub(496 hub_model_id,497 commit_message=self.commit_message,498 token=self.hf_token,499 )500 self.tokenizer.push_to_hub(501 hub_model_id,502 commit_message=self.commit_message, 503 token=self.hf_token,504 )505 506 # pushing the log files 507 # self.push_latest_tensorboard_log(logs_dir='logs', 508 # hf_model_repo=self.hub_model_id, 509 # hf_token= self.hf_token510 # )511 512 513 514 # Step 2: Generate model metadata515 metadata = {516 "model_name": hub_model_id,517 "self.version_tag": self.version_tag,518 "commit_message": commit_message,519 "timestamp_utc": timestamp,520 "author": "mr-kush",521 "training_args": self.sanitize_training_args,522 "eval_metrics": getattr(self, "classification_report", {}),523 }524 525 with open(metadata_path, "w") as f:526 json.dump(metadata, f, indent=4)527 528 # Step 3: Upload metadata to the Hub529 self.api.upload_file(530 path_or_fileobj=metadata_path,531 path_in_repo="model_metadata.json",532 repo_id=hub_model_id,533 repo_type="model",534 token=self.hf_token,535 commit_message= f"Upload model_metadata.json ({self.version_tag})"536 )537 538 # Step 4: Create version tag539 self.api.create_tag(540 repo_id=hub_model_id,541 repo_type="model",542 tag=self.version_tag,543 token=self.hf_token,544 )545 546 print(f"Model successfully pushed and tagged as {self.version_tag} on {hub_model_id}", flush=True)547 548 549 550 551 except Exception as e:552 print(f"Push failed: {e}", flush=True)553 554 finally:555 # Step 5: Clean up temporary metadata file556 if os.path.exists(metadata_path):557 try:558 os.remove(metadata_path)559 print("Temporary file model_metadata.json removed successfully.", flush=True)560 except Exception as cleanup_error:561 print(f"Warning: Could not delete model_metadata.json ({cleanup_error})", flush=True)562 563 def train_pipeline(564 self,565 train_dataset,566 eval_dataset,567 test_dataset,568 dataset_metadata: dict, 569 space_repo_id: str | None = None,570 hf_training_args: dict | None = None,571 api_endpoint: str | None = None,572 early_stopping_patience: int = 2,573 early_stopping_threshold: float = 0.001,574 deployed_sample_size: int = 300,575 decision_threshold: float = 0.001576 ):577 """578 Complete training, evaluation, decision-making, and optional auto-deployment pipeline.579 580 Args:581 train_dataset: Hugging Face Dataset for training.582 eval_dataset: Hugging Face Dataset for validation.583 test_dataset: Hugging Face Dataset for testing.584 dataset_metadata: Metadata about Data for Logging 585 hf_training_args (dict, optional): Hugging Face TrainingArguments overrides.586 api_endpoint (str, optional): Endpoint of deployed model to compare F1.587 space_repo_id (str): HF Space Repo Id. 588 early_stopping_patience (int): Patience for early stopping callback.589 early_stopping_threshold (float): Threshold for early stopping.590 deployed_sample_size (int): Sample size to query deployed model for comparison.591 decision_threshold (float): Minimum F1 improvement for auto-deploy.592 Returns:593 dict: Contains evaluation metrics, decision, and deployed F1 (if applicable).594 """595 self.space_repo_id= space_repo_id596 self.dataset_metadata = dataset_metadata597 598 # 1. Initialize W&B run599 wandb.init(600 project=self.wandb_project_name,601 name=f"train_pipeline_{datetime.now(UTC).strftime('%Y%m%d_%H%M%S')}",602 config={603 "model_checkpoint": self.model_checkpoint,604 "num_labels": self.num_labels,605 "dataset_metadata": self.dataset_metadata606 }607 )608 609 # 2. Train the model610 self.train(611 train_dataset=train_dataset,612 eval_dataset=eval_dataset,613 hf_training_args=hf_training_args,614 early_stopping_patience=early_stopping_patience,615 early_stopping_threshold=early_stopping_threshold616 )617 618 # Log sanitized training args to W&B config619 wandb.config.update(self.sanitize_training_args)620 621 # 3. Evaluate model on test dataset (no logging inside evaluate)622 eval_results = self.evaluate(623 test_dataset=test_dataset,624 api_endpoint=api_endpoint,625 threshold=decision_threshold,626 deployed_sample_size=deployed_sample_size627 )628 629 630 631 # 4. Extract outputs632 y_true = eval_results["y_true"]633 y_pred = eval_results["predictions"]634 cm = eval_results["confusion_matrix"]635 classification_report = eval_results["classification_report"]636 current_trained_f1_macro = eval_results["current_trained_f1_macro"]637 deployed_f1_macro = eval_results.get("deployed_f1_macro", None)638 decision = eval_results["decision"]639 640 # 5. Log evaluation metrics to W&B641 self.log_wandb_eval_metrics(642 y_true=y_true,643 y_pred=y_pred,644 classification_report_dict=classification_report,645 confusion_matrix_array=cm,646 label_names=list(self.id2label.values()),647 prefix="train_pipeline_eval"648 )649 650 # 6. Decision logic for auto-deployment651 deployed_f1_to_compare = deployed_f1_macro if deployed_f1_macro is not None else 0.0652 decision = "accepted" if current_trained_f1_macro > deployed_f1_to_compare + decision_threshold else "rejected"653 654 # 7. Log decision and F1 metrics to W&B655 wandb.log({656 "current_trained_model_f1_macro": current_trained_f1_macro,657 "deployed_model_f1_macro": deployed_f1_to_compare,658 "decision": decision,659 "timestamp": datetime.now(UTC).isoformat()660 })661 662 # 8. Tag run and summarize663 wandb.run.tags = ["train_pipeline", decision]664 wandb.run.summary["accepted"] = (decision == "accepted")665 666 667 # 9. Auto-deploy if decision accepted668 if decision == "accepted":669 try:670 # 9.1: push model to hub 671 self.push_model_to_hub(672 hub_model_id=self.hub_model_id,673 use_trainer=True,674 commit_message=f"Auto-deploy: ΔF1 >= {decision_threshold:.4f}"675 )676 677 # 9.2: restart th space678 self.restart_space(679 space_repo_id=self.space_repo_id680 )681 682 683 except Exception as e:684 wandb.log({"push_error": str(e)})685 raise RuntimeError(f"Warning: push to hub failed: {e}")686 687 # 10a. Send summary alert before finishing the run688 wandb.alert(689 title=f"Run Summary: {self.hub_model_id} ",690 text=(691 f"Decision: {decision}\n"692 f"Current Trained F1 Macro: {current_trained_f1_macro}\n"693 f"Deployed F1 Macro: {deployed_f1_macro}"694 ),695 level=AlertLevel.INFO,696 wait_duration=timedelta(minutes=1) # optional delay before sending697 )698 699 700 # 10.b Finish W&B run cleanly701 wandb.join()702 wandb.finish()703 704 # 11. Return outcomes705 return {706 "decision": decision,707 "current_trained_f1_macro": current_trained_f1_macro, 708 "deployed_f1": deployed_f1_macro,709 "classification_report": classification_report,710 "confusion_matrix": cm,711 "y_true": y_true,712 "y_pred": y_pred713 }714 715 716 717 718 719 def restart_space(self, 720 space_repo_id: str721 ):722 """723 Restarts the Hugging Face Space programmatically.724 725 Args:726 space_repo_id (str): HF Space Repo Id 727 728 Raises:729 ValueError: If 'repo_id' or 'token' is empty.730 RuntimeError: If the restart operation fails.731 """732 if not self.space_repo_id: 733 self.space_repo_id = space_repo_id734 735 if not self.space_repo_id or not self.hf_token:736 raise ValueError("Failed to Restart Space: Both 'repo_id' and 'token' must be provided.")737 738 try:739 self.api.restart_space(repo_id=self.space_repo_id,token=self.hf_token)740 print(f"Successfully restarted Space: {self.space_repo_id}", flush=True)741 except HfHubHTTPError as e:742 raise RuntimeError(f"Failed to restart Space '{self.space_repo_id}': {e}")743 except Exception as e:744 raise RuntimeError(f"An unexpected error occurred: {e}")745 746 747 # # def push_latest_tensorboard_log(self, logs_dir: str,748 # hf_model_repo: str,749 # hf_token: str,750 # runs_dir: str = "runs"):751 # """752 # Upload the latest TensorBoard event file from a logs directory753 # to a Hugging Face model repo under a TensorBoard-style folder.754 755 # Folder and file host IDs will match, e.g.:756 # runs/Sep26_05-06-52_0646998ee581/events.out.tfevents.<timestamp>.0646998ee581.<pid>.0757 # """758 759 # # Step 1: List all event files760 # event_files = [761 # f for f in os.listdir(logs_dir)762 # if f.startswith("events.out.tfevents")763 # ]764 765 # if not event_files:766 # print(f"No TensorBoard event files found in {logs_dir}.")767 # return768 769 # # Step 2: Find latest by modification time770 # latest_file = max(event_files, key=lambda f: os.path.getmtime(os.path.join(logs_dir, f)))771 # latest_file_path = os.path.join(logs_dir, latest_file)772 773 # # Step 3: Extract hostname part (index 4)774 # parts = latest_file.split('.')775 # if len(parts) >= 5:776 # host_id = parts[4] # e.g., '0646998ee581'777 # else:778 # # fallback: use system hostname779 # host_id = socket.gethostname()[:12]780 781 # # Step 4: Create TensorBoard-like folder name782 # timestamp = datetime.now(UTC).strftime("%b%d_%H-%M-%S")783 # new_folder_name = f"{timestamp}_{host_id}"784 785 # # Step 5: Construct path in HF repo786 # hf_upload_path = f"{runs_dir}/{new_folder_name}/{latest_file}"787 788 # # Step 6: Upload to Hugging Face Hub789 # self.api.upload_file(790 # path_or_fileobj=latest_file_path,791 # path_in_repo=hf_upload_path,792 # repo_id=hf_model_repo,793 # repo_type="model",794 # token=hf_token,795 # commit_message=f"Upload latest TensorBoard log: {latest_file}"796 # )797 798 # print(f"Uploaded '{latest_file}' → '{hf_model_repo}/{hf_upload_path}'")799 800 # # Step 7: Delete local event file 801 # try:802 # os.remove(latest_file_path)803 # print(f"Cleared local file: {latest_file_path}")804 # except Exception as e:805 # print(f"Could not delete file '{latest_file_path}': {e}")806 807