NLPGenius/CVE-FactChecker

1#!/usr/bin/env python32"""3Firebase collection explorer to understand the actual database structure.4"""5 6import os7import sys8import requests9 10# Add the parent directory to Python path11current_dir = os.path.dirname(os.path.abspath(__file__))12sys.path.insert(0, current_dir)13 14def explore_firebase_collections():15    """Explore available Firebase collections and their structure."""16    print("🔍 Firebase Collections Explorer")17    print("=" * 60)18    19    try:20        from cve_factchecker.firebase_loader import FirebaseNewsLoader21        22        loader = FirebaseNewsLoader()23        project_id = loader.project_id24        api_key = loader.config.api_key25        26        print(f"📡 Project ID: {project_id}")27        28        # Try different collection names29        collection_candidates = [30            "articles",31            "english_articles", 32            "Articles",33            "English_articles",34            "news_articles",35            "cve_articles",36            "documents"37        ]38        39        found_collections = []40        41        for collection_name in collection_candidates:42            print(f"\n🔍 Checking collection: '{collection_name}'")43            44            try:45                base_url = f"https://firestore.googleapis.com/v1/projects/{project_id}/databases/(default)/documents/{collection_name}"46                params = {47                    "key": api_key,48                    "pageSize": 3  # Small sample49                }50                51                resp = requests.get(base_url, params=params, timeout=15)52                53                if resp.status_code == 200:54                    data = resp.json()55                    docs = data.get("documents", [])56                    57                    if docs:58                        print(f"   ✅ Found {len(docs)} documents")59                        found_collections.append(collection_name)60                        61                        # Analyze first document62                        first_doc = docs[0]63                        if "fields" in first_doc:64                            fields = first_doc["fields"]65                            field_names = list(fields.keys())66                            print(f"   📊 Fields: {field_names}")67                            68                            # Check for language field69                            if "language" in fields:70                                lang_value = fields["language"]71                                if "stringValue" in lang_value:72                                    print(f"   🌐 Language: '{lang_value['stringValue']}'")73                            74                            # Check for content fields75                            content_fields = [f for f in field_names if any(term in f.lower() for term in ['content', 'text', 'article'])]76                            if content_fields:77                                print(f"   📝 Content fields: {content_fields}")78                                79                                # Show content sample80                                for cf in content_fields[:1]:  # First content field81                                    if cf in fields and "stringValue" in fields[cf]:82                                        content_sample = fields[cf]["stringValue"][:100]83                                        print(f"   📖 {cf} sample: {content_sample}...")84                    else:85                        print(f"   📭 Collection exists but is empty")86                        found_collections.append(f"{collection_name} (empty)")87                        88                elif resp.status_code == 404:89                    print(f"   ❌ Collection does not exist")90                else:91                    print(f"   ⚠️ Error {resp.status_code}: {resp.text[:100]}")92                    93            except Exception as e:94                print(f"   ❌ Error checking collection: {e}")95        96        print(f"\n📋 Summary:")97        print(f"   Found collections: {found_collections}")98        99        # If main articles collection exists, explore language distribution100        if "articles" in found_collections:101            print(f"\n🌐 Analyzing language distribution in 'articles' collection...")102            explore_language_distribution(loader, "articles")103        104        return found_collections105        106    except Exception as e:107        print(f"❌ Firebase exploration failed: {e}")108        import traceback109        traceback.print_exc()110        return []111 112def explore_language_distribution(loader, collection_name, sample_size=10):113    """Explore language distribution in a collection."""114    try:115        base_url = f"https://firestore.googleapis.com/v1/projects/{loader.project_id}/databases/(default)/documents/{collection_name}"116        params = {117            "key": loader.config.api_key,118            "pageSize": sample_size119        }120        121        resp = requests.get(base_url, params=params, timeout=15)122        123        if resp.status_code == 200:124            data = resp.json()125            docs = data.get("documents", [])126            127            language_counts = {}128            content_lengths = []129            130            for doc in docs:131                if "fields" in doc:132                    fields = doc["fields"]133                    134                    # Check language135                    lang = "unknown"136                    if "language" in fields and "stringValue" in fields["language"]:137                        lang = fields["language"]["stringValue"]138                    139                    language_counts[lang] = language_counts.get(lang, 0) + 1140                    141                    # Check content length142                    content_fields = ["content", "Content", "article_text", "Article_text", "text"]143                    for cf in content_fields:144                        if cf in fields and "stringValue" in fields[cf]:145                            content_length = len(fields[cf]["stringValue"])146                            content_lengths.append(content_length)147                            break148            149            print(f"   Language distribution: {language_counts}")150            if content_lengths:151                avg_length = sum(content_lengths) / len(content_lengths)152                print(f"   Average content length: {avg_length:.0f} characters")153                print(f"   Content range: {min(content_lengths)} - {max(content_lengths)} characters")154        155    except Exception as e:156        print(f"   ❌ Error analyzing language distribution: {e}")157 158def create_test_collection_strategy(found_collections):159    """Create a strategy for testing based on found collections."""160    print(f"\n💡 Recommended Testing Strategy")161    print("=" * 60)162    163    if "articles" in found_collections:164        print("✅ Use 'articles' collection with language filtering")165        print("   - This appears to be the main collection")166        print("   - Filter by language='English' or similar")167        168        # Test language filtering169        print(f"\n🧪 Testing language filtering on 'articles' collection...")170        test_language_filtering()171        172    elif any("english" in col.lower() for col in found_collections):173        english_collections = [col for col in found_collections if "english" in col.lower()]174        print(f"✅ Use English-specific collection: {english_collections[0]}")175        176    else:177        print("⚠️ No obvious English collection found")178        print("💡 Recommended approach:")179        print("   1. Use the largest available collection")180        print("   2. Apply content-based English detection")181        182    return found_collections183 184def test_language_filtering():185    """Test different language filter values."""186    try:187        from cve_factchecker.firebase_loader import FirebaseNewsLoader188        189        loader = FirebaseNewsLoader()190        191        # Test different language values192        language_variants = ["English", "english", "en", "EN", "eng"]193        194        for lang in language_variants:195            print(f"   Testing language='{lang}'...")196            articles = loader.fetch_articles(limit=5, language=lang)197            print(f"     Result: {len(articles)} articles")198            199            if articles:200                # Show sample201                sample = articles[0]202                print(f"     Sample: {sample.title[:50]}...")203                break204                205    except Exception as e:206        print(f"   ❌ Language filtering test failed: {e}")207 208def main():209    """Main exploration function."""210    print("🔍 CVE Fact Checker - Firebase Database Explorer")211    print("=" * 80)212    213    found_collections = explore_firebase_collections()214    215    if found_collections:216        create_test_collection_strategy(found_collections)217    else:218        print("❌ No collections found. Check Firebase configuration.")219    220    return bool(found_collections)221 222if __name__ == "__main__":223    success = main()224    sys.exit(0 if success else 1)