Premchan369/Q-TensorFormer

1"""2src/validator.py3Scientific Claim Validator & Empirical Consistency Auditor for Q-TensorFormer.4 5Enforces empirical integrity across all research outputs:6  1. Validates classification of every reported metric:7      MEASURED: Obtained directly from hardware timing/profiling.8      ESTIMATED: Computed using explicit, calibrated models.9      SIMULATED: Run on classical quantum statevector simulator.10      PROJECTED: Theoretical analytical scale-up calculation.11  2. Flags and rejects:12      - Unsupported "quantum advantage" claims without real QPU hardware execution.13      - Latency claims derived solely from FLOP counts.14      - "Zero runtime overhead" or "zero overhead" claims (slicing overhead ~20-50 us is empirically verified).15      - Unproven asymptotic guarantees (e.g. "asymptotic convergence guaranteed").16      - Fabricated benchmark percentages.17  3. Audits output artifact completeness:18      Verifies that all 13 core JSON result files and 14 publication figure PNGs exist.19"""20 21import json22import re23import sys24import os25from pathlib import Path26from typing import Dict, List, Tuple, Any, Set, Optional27 28 29class ScientificClaimValidator:30    """31    Automated validator for research claims, benchmark tables, and model card disclosures.32    """33 34    ALLOWED_CLASSIFICATIONS = {"MEASURED", "ESTIMATED", "SIMULATED", "PROJECTED"}35 36    FORBIDDEN_PHRASES = [37        "quantum advantage in nlp",38        "quantum speedup on cpu",39        "first ever adaptive transformer",40        "first entropy-based transformer",41        "zero latency overhead with quantum simulation",42        "zero runtime overhead",43        "asymptotic convergence to the exact boundary",44        "asymptotic pareto optimality guaranteed",45    ]46 47    REQUIRED_JSON_ARTIFACTS = [48        "marginal_value_results.json",49        "information_state_ablation.json",50        "adaptive_rank_results.json",51        "nested_tt_analysis.json",52        "gqa_audit.json",53        "kv_rate_distortion.json",54        "controller_convergence.json",55        "hysteresis_results.json",56        "quantum_utility.json",57        "detailed_latency_profile.json",58        "pareto_frontiers.json",59        "counter_hypothesis_results.json",60        "comprehensive_comparison.json",61        "baseline_comparison_master.json",62        "counterfactual_learning_results.json",63        "hierarchical_kv_results.json",64        "phase_profiling_results.json",65        "matched_budget_evaluations.json",66        "workload_adaptation_results.json",67        "quantum_utility_boundary.json",68        "scaling_projections.json",69    ]70 71    REQUIRED_FIGURE_ARTIFACTS = [72        "figure1_architecture.png",73        "figure2_marginal_value_r2.png",74        "figure3_8d_lofo_importance.png",75        "figure4_nested_tt_suboptimality.png",76        "figure5_adaptive_rank_latency_traffic.png",77        "figure6_gqa_memory_traffic.png",78        "figure7_kv_rate_distortion.png",79        "figure8_controller_convergence.png",80        "figure9_hysteresis_churn_jitter.png",81        "figure10_quantum_utility_tradeoff.png",82        "figure11_subsystem_latency_breakdown.png",83        "figure12_hardware_roofline.png",84        "figure13_multi_pareto_frontiers.png",85        "figure14_counter_hypothesis_boundaries.png",86        "figure15_baseline_pareto_frontiers.png",87        "figure16_baseline_improvement_radar.png",88    ]89 90    def __init__(self, root_dir: Optional[Path] = None):91        self.root_dir = root_dir or Path(__file__).parent.parent92        self.validation_errors: List[str] = []93        self.warnings: List[str] = []94 95    def validate_metric_record(self, record: Dict[str, Any]) -> bool:96        """Validate a single benchmark metric entry."""97        metric_name = record.get("metric", "unknown")98        classification = record.get("classification", "").upper()99 100        if classification not in self.ALLOWED_CLASSIFICATIONS:101            self.validation_errors.append(102                f"Metric '{metric_name}' has invalid classification '{classification}'. "103                f"Must be one of: {self.ALLOWED_CLASSIFICATIONS}"104            )105            return False106 107        if "latency" in metric_name.lower() and classification == "MEASURED" and "hardware" not in record:108            self.warnings.append(109                f"Measured latency '{metric_name}' should specify hardware device used for measurement."110            )111 112        return True113 114    def validate_document_text(self, text: str, doc_name: str = "Document") -> bool:115        """Scan text for forbidden/unsubstantiated claims."""116        clean_text = text.lower()117        passed = True118 119        for phrase in self.FORBIDDEN_PHRASES:120            if phrase in clean_text:121                self.validation_errors.append(122                    f"[{doc_name}] Found unsubstantiated claim phrase: '{phrase}'"123                )124                passed = False125 126        return passed127 128    def validate_artifacts(self) -> bool:129        """Audit that all empirical outputs and publication figures exist and are valid."""130        outputs_dir = self.root_dir / "outputs"131        figures_dir = outputs_dir / "figures"132        passed = True133 134        if not outputs_dir.exists():135            self.validation_errors.append(f"Outputs directory {outputs_dir} does not exist.")136            return False137 138        # 1. Audit JSON files139        for j_name in self.REQUIRED_JSON_ARTIFACTS:140            j_path = outputs_dir / j_name141            if not j_path.exists():142                self.validation_errors.append(f"Missing required empirical artifact: outputs/{j_name}")143                passed = False144            else:145                try:146                    with open(j_path, "r", encoding="utf-8") as f:147                        data = json.load(f)148                        if not data:149                            self.validation_errors.append(f"Empty artifact: outputs/{j_name}")150                            passed = False151                except Exception as e:152                    self.validation_errors.append(f"Corrupt JSON in outputs/{j_name}: {e}")153                    passed = False154 155        # 2. Audit Figures156        if not figures_dir.exists():157            self.validation_errors.append(f"Figures directory {figures_dir} does not exist.")158            return False159 160        for f_name in self.REQUIRED_FIGURE_ARTIFACTS:161            f_path = figures_dir / f_name162            if not f_path.exists():163                self.validation_errors.append(f"Missing required figure artifact: outputs/figures/{f_name}")164                passed = False165            elif f_path.stat().st_size < 1000:166                self.validation_errors.append(f"Figure {f_name} is too small (< 1KB), possible blank render.")167                passed = False168 169        return passed170 171    def generate_report(self) -> Dict[str, Any]:172        return {173            "status": "FAILED" if self.validation_errors else "PASSED",174            "errors": self.validation_errors,175            "warnings": self.warnings,176            "total_errors": len(self.validation_errors),177            "total_warnings": len(self.warnings),178            "artifacts_verified": len(self.REQUIRED_JSON_ARTIFACTS) + len(self.REQUIRED_FIGURE_ARTIFACTS),179        }180 181 182def main():183    root_dir = Path(__file__).parent.parent184    validator = ScientificClaimValidator(root_dir)185 186    # 1. Audit artifacts187    validator.validate_artifacts()188 189    # 2. Audit documents190    for doc in ["README.md", "MODEL_CARD.md", "docs/CLAIM_TO_CODE_MAP.md", "docs/ARCHITECTURE_AUDIT.md"]:191        p = root_dir / doc192        if p.exists():193            content = p.read_text(encoding="utf-8")194            validator.validate_document_text(content, doc_name=doc)195 196    report = validator.generate_report()197    print("=" * 70)198    print("SCIENTIFIC CLAIM & ARTIFACT CONSISTENCY VALIDATION REPORT")199    print("=" * 70)200    print(f"Status: {report['status']}")201    print(f"Verified Artifacts: {report['artifacts_verified']} files")202    print(f"Errors ({report['total_errors']}):")203    for err in report["errors"]:204        print(f"  [ERROR] {err}")205    print(f"Warnings ({report['total_warnings']}):")206    for warn in report["warnings"]:207        print(f"  [WARN] {warn}")208    print("=" * 70)209 210    if report["status"] == "FAILED":211        sys.exit(1)212    else:213        print("[PASSED] All scientific claims and empirical artifacts verified successfully!")214 215 216if __name__ == "__main__":217    from typing import Optional218    main()219