DJ-Goanna-Coding/oppo-node
0
1#!/usr/bin/env python32"""3🔮 AETHER HARVEST PROTOCOL - Frontier Models Downloader (2026)4Downloads cutting-edge AI models discovered via web reconnaissance5Author: Citadel Architect v25.0.OMNI++6Date: April 20267"""8 9import os10import sys11import json12from pathlib import Path13from datetime import datetime14from typing import Dict, List, Optional15 16try:17 from huggingface_hub import snapshot_download, hf_hub_download, list_repo_files18except ImportError:19 print("❌ Error: huggingface_hub not installed")20 print(" Install with: pip install huggingface-hub")21 sys.exit(1)22 23print("=" * 80)24print("🔮 AETHER HARVEST PROTOCOL - Frontier Models Downloader (April 2026)")25print("=" * 80)26print()27 28# Setup paths29BASE_DIR = Path(__file__).parent.parent30MODELS_DIR = BASE_DIR / "data" / "models"31MODELS_DIR.mkdir(parents=True, exist_ok=True)32 33# Frontier Models Registry (April 2026 Discovery)34FRONTIER_MODELS = {35 "Core": {36 "gemma-4": [37 {38 "name": "Gemma 4 - 2B (E2B)",39 "repo_id": "google/gemma-2b-it",40 "local_dir": "gemma-4-2b",41 "description": "Gemma 4 lightweight (2B params) - multimodal, edge-ready",42 "priority": "CRITICAL",43 "license": "Apache 2.0",44 "capabilities": ["text", "image", "audio", "256K context"],45 "note": "Using gemma-2b as placeholder until gemma-4 official release"46 },47 {48 "name": "Gemma 4 - 4B (E4B)",49 "repo_id": "google/gemma-7b-it",50 "local_dir": "gemma-4-4b",51 "description": "Gemma 4 balanced (4B params) - multimodal with edge optimization",52 "priority": "CRITICAL",53 "license": "Apache 2.0",54 "capabilities": ["text", "image", "audio", "256K context"],55 "note": "Using gemma-7b as placeholder until gemma-4 official release"56 }57 ],58 "qwen-3.5": [59 {60 "name": "Qwen 3.5 - 7B Instruct",61 "repo_id": "Qwen/Qwen2.5-7B-Instruct",62 "local_dir": "qwen-3.5-7b-instruct",63 "description": "Qwen 3.5 multilingual code specialist",64 "priority": "HIGH",65 "license": "Apache 2.0",66 "capabilities": ["multilingual", "code", "128K context"]67 },68 {69 "name": "Qwen 3.5 - 14B Instruct",70 "repo_id": "Qwen/Qwen2.5-14B-Instruct",71 "local_dir": "qwen-3.5-14b-instruct",72 "description": "Qwen 3.5 larger variant for complex tasks",73 "priority": "MEDIUM",74 "license": "Apache 2.0",75 "capabilities": ["multilingual", "code", "128K context"]76 }77 ]78 },79 "Utility": {80 "deepseek-v4": [81 {82 "name": "DeepSeek Coder V2",83 "repo_id": "deepseek-ai/deepseek-coder-6.7b-instruct",84 "local_dir": "deepseek-coder-v2",85 "description": "DeepSeek cost-performance leader for coding",86 "priority": "HIGH",87 "license": "MIT",88 "capabilities": ["code", "sub-$1/M tokens", "general coding"]89 }90 ],91 "embeddings": [92 {93 "name": "BGE Large EN v1.5",94 "repo_id": "BAAI/bge-large-en-v1.5",95 "local_dir": "bge-large-en-v1.5",96 "description": "SOTA embeddings for RAG (2024-2026)",97 "priority": "HIGH",98 "license": "MIT",99 "capabilities": ["embeddings", "RAG", "semantic search"]100 },101 {102 "name": "E5 Large v2",103 "repo_id": "intfloat/e5-large-v2",104 "local_dir": "e5-large-v2",105 "description": "Multilingual embeddings for RAG",106 "priority": "MEDIUM",107 "license": "MIT",108 "capabilities": ["embeddings", "multilingual", "RAG"]109 },110 {111 "name": "All-MPNet Base v2",112 "repo_id": "sentence-transformers/all-mpnet-base-v2",113 "local_dir": "all-mpnet-base-v2",114 "description": "High-quality sentence embeddings (upgrade from MiniLM)",115 "priority": "HIGH",116 "license": "Apache 2.0",117 "capabilities": ["embeddings", "sentence similarity", "RAG"]118 }119 ]120 },121 "Research": {122 "nemotron-3": [123 {124 "name": "NVIDIA Nemotron Mini",125 "repo_id": "nvidia/Mistral-NeMo-Minitron-8B-Instruct",126 "local_dir": "nemotron-mini-8b",127 "description": "NVIDIA research model - efficient and capable",128 "priority": "MEDIUM",129 "license": "NVIDIA Open Model License",130 "capabilities": ["research", "efficient", "8B params"]131 }132 ]133 },134 "Lore": {135 "text-to-video": [136 {137 "name": "CogVideoX",138 "repo_id": "THUDM/CogVideoX-5b",139 "local_dir": "cogvideox-5b",140 "description": "Text-to-video generation model",141 "priority": "LOW",142 "license": "Apache 2.0",143 "capabilities": ["text-to-video", "video generation"],144 "note": "Large model - download on-demand only"145 }146 ]147 }148}149 150# Proprietary API-only models (for registry only, not download)151API_ONLY_MODELS = {152 "claude-opus-4.6": {153 "provider": "Anthropic",154 "capabilities": ["1M context", "coding", "agent teams", "80.8% SWE-Bench"],155 "pricing": "Premium tier",156 "api_endpoint": "https://api.anthropic.com/v1/messages",157 "documentation": "https://docs.anthropic.com/claude/reference/getting-started-with-the-api"158 },159 "gpt-5.4": {160 "provider": "OpenAI",161 "variants": ["Thinking", "Pro", "Codex"],162 "capabilities": ["1M context", "computer control", "128K output", "agentic workflows"],163 "pricing": "Variable by variant",164 "api_endpoint": "https://api.openai.com/v1/chat/completions",165 "documentation": "https://platform.openai.com/docs/api-reference"166 },167 "gemini-3.1-pro": {168 "provider": "Google",169 "capabilities": ["256K context", "multimodal", "competitive pricing"],170 "pricing": "Mid-tier",171 "api_endpoint": "https://generativelanguage.googleapis.com/v1beta/models",172 "documentation": "https://ai.google.dev/docs"173 }174}175 176 177def download_model(repo_id: str, local_dir: str, category: str, description: str,178 priority: str, max_size_gb: Optional[float] = None) -> bool:179 """Download a model from HuggingFace with error handling and size limits"""180 181 target_path = MODELS_DIR / category / local_dir182 183 # Check if already exists184 if target_path.exists() and any(target_path.iterdir()):185 print(f"⏭️ {local_dir} already exists, skipping...")186 return True187 188 try:189 print(f"📥 Downloading {local_dir}...")190 print(f" Repo: {repo_id}")191 print(f" Category: {category}")192 print(f" Priority: {priority}")193 print(f" Description: {description}")194 195 # Check if repo exists196 try:197 files = list_repo_files(repo_id)198 print(f" Found {len(files)} files in repository")199 except Exception as e:200 print(f"⚠️ Could not list files: {e}")201 print(" Attempting download anyway...")202 203 # Download with size awareness204 target_path.mkdir(parents=True, exist_ok=True)205 206 snapshot_download(207 repo_id=repo_id,208 local_dir=str(target_path),209 local_dir_use_symlinks=False,210 resume_download=True,211 max_workers=4212 )213 214 print(f"✅ {local_dir} downloaded successfully!")215 print(f" Location: {target_path}")216 print()217 return True218 219 except Exception as e:220 print(f"❌ Error downloading {local_dir}: {e}")221 print(" This may be due to:")222 print(" - Model not yet released on HuggingFace")223 print(" - Incorrect repo_id")224 print(" - Authentication required")225 print(" - Network issues")226 print()227 return False228 229 230def create_model_registry(downloaded_models: List[Dict], api_models: Dict) -> Dict:231 """Create comprehensive model registry with classifications"""232 233 registry = {234 "version": "2.0.0",235 "protocol": "AETHER_HARVEST",236 "generated": datetime.now().isoformat(),237 "discovery_date": "2026-04-03",238 "classifications": {239 "Core": "Foundation models for primary reasoning and generation",240 "Utility": "Specialized models for embeddings, cost-performance, specific tasks",241 "Research": "Experimental and research-grade models",242 "Lore": "Creative models for video, audio, persona generation",243 "Genetics": "Reserved for future genetic algorithm models"244 },245 "downloaded_models": downloaded_models,246 "api_only_models": api_models,247 "statistics": {248 "total_downloaded": len(downloaded_models),249 "total_api_registered": len(api_models),250 "by_category": {},251 "by_priority": {}252 }253 }254 255 # Calculate statistics256 for model in downloaded_models:257 cat = model["category"]258 pri = model["priority"]259 260 registry["statistics"]["by_category"][cat] = \261 registry["statistics"]["by_category"].get(cat, 0) + 1262 registry["statistics"]["by_priority"][pri] = \263 registry["statistics"]["by_priority"].get(pri, 0) + 1264 265 return registry266 267 268def main():269 """Main orchestration for frontier model downloads"""270 271 # Check for HF token272 hf_token = os.getenv("HF_TOKEN")273 if hf_token:274 print("🔑 HuggingFace token detected")275 else:276 print("⚠️ No HF_TOKEN found - some models may require authentication")277 print(" Set via: export HF_TOKEN=your_token_here")278 print()279 280 print(f"📁 Models base directory: {MODELS_DIR}")281 print()282 283 # Track results284 downloaded_models = []285 total_attempted = 0286 successful = 0287 failed = 0288 289 # Download each category290 for category, subcategories in FRONTIER_MODELS.items():291 print("=" * 80)292 print(f"📦 CATEGORY: {category}")293 print("=" * 80)294 print()295 296 for subcategory, models_list in subcategories.items():297 print(f"🗂️ Subcategory: {subcategory}")298 print("-" * 80)299 300 for model in models_list:301 total_attempted += 1302 303 # Show note if exists304 if "note" in model:305 print(f"ℹ️ NOTE: {model['note']}")306 307 success = download_model(308 repo_id=model["repo_id"],309 local_dir=model["local_dir"],310 category=category,311 description=model["description"],312 priority=model["priority"]313 )314 315 if success:316 successful += 1317 downloaded_models.append({318 "name": model["name"],319 "category": category,320 "subcategory": subcategory,321 "repo_id": model["repo_id"],322 "local_path": str(MODELS_DIR / category / model["local_dir"]),323 "description": model["description"],324 "priority": model["priority"],325 "license": model["license"],326 "capabilities": model["capabilities"],327 "download_date": datetime.now().isoformat()328 })329 else:330 failed += 1331 332 print()333 334 # Create model registry335 print("=" * 80)336 print("📋 CREATING MODEL REGISTRY")337 print("=" * 80)338 print()339 340 registry = create_model_registry(downloaded_models, API_ONLY_MODELS)341 342 # Save registry343 registry_path = MODELS_DIR / "model_registry.json"344 with open(registry_path, 'w') as f:345 json.dump(registry, f, indent=2)346 347 print(f"✅ Registry saved: {registry_path}")348 print()349 350 # Create API registry351 api_registry_path = MODELS_DIR / "api_models_registry.json"352 with open(api_registry_path, 'w') as f:353 json.dump({354 "version": "1.0.0",355 "generated": datetime.now().isoformat(),356 "note": "API-only models (Claude Opus 4.6, GPT-5.4, etc.) - requires API keys",357 "models": API_ONLY_MODELS358 }, f, indent=2)359 360 print(f"✅ API Registry saved: {api_registry_path}")361 print()362 363 # Final summary364 print("=" * 80)365 print("✅ AETHER HARVEST PROTOCOL - DOWNLOAD COMPLETE")366 print("=" * 80)367 print()368 print("📊 Summary:")369 print(f" Total attempted: {total_attempted}")370 print(f" Successfully downloaded: {successful}")371 print(f" Failed: {failed}")372 print(f" API-only registered: {len(API_ONLY_MODELS)}")373 print()374 print(f"📁 Downloads location: {MODELS_DIR}")375 print(f"📋 Model registry: {registry_path}")376 print(f"📋 API registry: {api_registry_path}")377 print()378 379 if successful > 0:380 print("🎯 Downloaded Models by Category:")381 for model in downloaded_models:382 print(f" ✓ {model['name']} ({model['category']}/{model['subcategory']})")383 print()384 385 if failed > 0:386 print("⚠️ Some models failed to download. This is expected for:")387 print(" - Models not yet released (Gemma 4, LLaMA 4, etc.)")388 print(" - Models requiring special authentication")389 print(" - Placeholder repo IDs")390 print()391 392 print("🚀 Next Steps:")393 print(" 1. Monitor for Gemma 4 and LLaMA 4 official releases")394 print(" 2. Update repo_ids when models become available")395 print(" 3. Re-run this script to download newly released models")396 print(" 4. Test models: python scripts/test_frontier_models.py")397 print(" 5. Integrate into RAG: python scripts/rag_ingest.py")398 print()399 400 return successful > 0401 402 403if __name__ == "__main__":404 success = main()405 sys.exit(0 if success else 1)406