videoloc/seamless-crossattention

1#!/usr/bin/env python32# Example usage for videoloc/seamless-crossattention3 4from transformers import AutoModel, AutoConfig5from huggingface_hub import hf_hub_download6import torch7import numpy as np8import importlib.util9 10def load_model_and_collator():11    # Load model - custom architecture requires importing the model class12    model_files = hf_hub_download(repo_id="videoloc/seamless-crossattention", filename="modeling_seamless_crossattention.py")13    spec = importlib.util.spec_from_file_location("modeling_seamless_crossattention", model_files)14    modeling_module = importlib.util.module_from_spec(spec)15    spec.loader.exec_module(modeling_module)16 17    # Now load the model using the custom class18    config = modeling_module.SeamlessCrossAttentionConfig.from_pretrained("videoloc/seamless-crossattention")19    model = modeling_module.HFSeamlessCrossAttention.from_pretrained("videoloc/seamless-crossattention")20    21    # Load data collator22    collator_file = hf_hub_download(repo_id="videoloc/seamless-crossattention", filename="data_collator.py")23    spec = importlib.util.spec_from_file_location("data_collator", collator_file)24    collator_module = importlib.util.module_from_spec(spec)25    spec.loader.exec_module(collator_module)26    27    data_collator = collator_module.DataCollatorSimpleSeamless(28        processor="facebook/hf-seamless-m4t-medium",29        max_audio_length_sec=8.0,30        max_text_length=25631    )32    33    return model, data_collator34 35def example_inference():36    model, collator = load_model_and_collator()37    38    # Example data: audio segment + subtitle text for cross-attention TTE prediction39    data = [{40        'raw_audio': np.random.randn(16000 * 3),  # 3 seconds at 16kHz41        'raw_text': "Example subtitle text with cross-modal attention for TTE prediction",42    }]43    44    batch = collator(data)45    model.eval()46    with torch.no_grad():47        outputs = model(**batch)48        tte_prediction = outputs.logits.item()49    50    print(f"Predicted Time To Edit (TTE): {tte_prediction:.2f} seconds")51    return tte_prediction52 53if __name__ == "__main__":54    example_inference()55