afeng/tokenizers

1#!/usr/bin/env python32"""3Simple test script to verify tokenizer functionality.4This tests the core functions without launching the Gradio interface.5"""6 7import sys8import json9 10# Test imports11try:12    from transformers import AutoTokenizer13    print("✓ transformers imported successfully")14except ImportError as e:15    print(f"✗ Failed to import transformers: {e}")16    sys.exit(1)17 18try:19    import gradio as gr20    print("✓ gradio imported successfully")21except ImportError as e:22    print(f"✗ Failed to import gradio: {e}")23    sys.exit(1)24 25# Test basic tokenization26def test_basic_tokenization():27    """Test basic tokenization with a small model."""28    print("\n--- Testing Basic Tokenization ---")29    try:30        # Use GPT-2 as it's small and commonly available31        model_id = "openai-community/gpt2"32        text = "Hello, world! This is a test."33 34        print(f"Loading tokenizer: {model_id}")35        tokenizer = AutoTokenizer.from_pretrained(model_id)36        print("✓ Tokenizer loaded successfully")37 38        # Test encoding39        encoded = tokenizer.encode(text)40        print(f"✓ Text encoded: {encoded[:10]}...")  # Show first 10 tokens41 42        # Test decoding43        decoded = tokenizer.decode(encoded)44        print(f"✓ Text decoded: {decoded}")45 46        # Verify round-trip47        assert decoded == text, "Round-trip tokenization failed"48        print("✓ Round-trip tokenization successful")49 50        # Test token conversion51        tokens = tokenizer.convert_ids_to_tokens(encoded)52        print(f"✓ Tokens: {tokens[:5]}...")  # Show first 5 tokens53 54        return True55    except Exception as e:56        print(f"✗ Test failed: {e}")57        return False58 59def test_special_tokens():60    """Test special token handling."""61    print("\n--- Testing Special Tokens ---")62    try:63        model_id = "openai-community/gpt2"64        text = "Test text"65 66        tokenizer = AutoTokenizer.from_pretrained(model_id)67 68        # With special tokens69        encoded_with = tokenizer.encode(text, add_special_tokens=True)70        # Without special tokens71        encoded_without = tokenizer.encode(text, add_special_tokens=False)72 73        print(f"✓ With special tokens: {len(encoded_with)} tokens")74        print(f"✓ Without special tokens: {len(encoded_without)} tokens")75 76        # Decode with and without special tokens77        decoded_with = tokenizer.decode(encoded_with, skip_special_tokens=False)78        decoded_without = tokenizer.decode(encoded_with, skip_special_tokens=True)79 80        print(f"✓ Decoded with special: {decoded_with}")81        print(f"✓ Decoded without special: {decoded_without}")82 83        return True84    except Exception as e:85        print(f"✗ Test failed: {e}")86        return False87 88def test_app_functions():89    """Test the main app functions."""90    print("\n--- Testing App Functions ---")91    try:92        # Import app functions93        from app import tokenize_text, decode_tokens, analyze_vocabulary94 95        # Test tokenize_text96        print("Testing tokenize_text function...")97        result = tokenize_text(98            text="Hello world",99            model_id="openai-community/gpt2",100            add_special_tokens=True,101            show_special_tokens=True,102            custom_model_id=None103        )104        assert len(result) == 5, "tokenize_text should return 5 values"105        print("✓ tokenize_text function works")106 107        # Test decode_tokens108        print("Testing decode_tokens function...")109        decode_result = decode_tokens(110            token_ids_str="[15496, 11, 995]",  # "Hello, world" in GPT-2111            model_id="openai-community/gpt2",112            skip_special_tokens=False,113            custom_model_id=None114        )115        assert "Decoded Text:" in decode_result, "decode_tokens should return decoded text"116        print("✓ decode_tokens function works")117 118        # Test analyze_vocabulary119        print("Testing analyze_vocabulary function...")120        vocab_result = analyze_vocabulary(121            model_id="openai-community/gpt2",122            custom_model_id=None123        )124        assert "Vocabulary Size:" in vocab_result, "analyze_vocabulary should return vocabulary info"125        print("✓ analyze_vocabulary function works")126 127        return True128    except Exception as e:129        print(f"✗ Test failed: {e}")130        import traceback131        traceback.print_exc()132        return False133 134def main():135    """Run all tests."""136    print("=" * 50)137    print("Tokenizer Playground Test Suite")138    print("=" * 50)139 140    tests = [141        test_basic_tokenization,142        test_special_tokens,143        test_app_functions144    ]145 146    results = []147    for test in tests:148        results.append(test())149 150    print("\n" + "=" * 50)151    print("Test Summary")152    print("=" * 50)153    passed = sum(results)154    total = len(results)155    print(f"Passed: {passed}/{total}")156 157    if passed == total:158        print("✅ All tests passed!")159        return 0160    else:161        print("❌ Some tests failed")162        return 1163 164if __name__ == "__main__":165    sys.exit(main())