CompileError/WebCoderBench

1from dataclasses import dataclass, field, make_dataclass2from enum import Enum3 4from src.about import Tasks5 6 7def fields(raw_class):8    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]9 10 11# These classes are for user facing column names,12# to avoid having to change them all around the code13# when a modif is needed14@dataclass15class ColumnContent:16    name: str17    type: str18    displayed_by_default: bool19    hidden: bool = False20    never_hidden: bool = False21 22 23## Leaderboard columns24auto_eval_column_dict = []25# Init26auto_eval_column_dict.append(["model_type_symbol", ColumnContent, field(default_factory=lambda: ColumnContent("T", "str", True, never_hidden=True))])27auto_eval_column_dict.append(["model", ColumnContent, field(default_factory=lambda: ColumnContent("Model", "markdown", True, never_hidden=True))])28# Scores29auto_eval_column_dict.append(["average", ColumnContent, field(default_factory=lambda: ColumnContent("Average ⬆️", "number", True))])30for task in Tasks:31    auto_eval_column_dict.append([task.name, ColumnContent, field(default_factory=lambda t=task: ColumnContent(t.value.col_name, "number", True))])32# Model information33auto_eval_column_dict.append(["model_type", ColumnContent, field(default_factory=lambda: ColumnContent("Type", "str", False))])34auto_eval_column_dict.append(["architecture", ColumnContent, field(default_factory=lambda: ColumnContent("Architecture", "str", False))])35auto_eval_column_dict.append(["weight_type", ColumnContent, field(default_factory=lambda: ColumnContent("Weight type", "str", False, True))])36auto_eval_column_dict.append(["precision", ColumnContent, field(default_factory=lambda: ColumnContent("Precision", "str", False))])37auto_eval_column_dict.append(["license", ColumnContent, field(default_factory=lambda: ColumnContent("Hub License", "str", False))])38auto_eval_column_dict.append(["params", ColumnContent, field(default_factory=lambda: ColumnContent("#Params (B)", "number", False))])39auto_eval_column_dict.append(["likes", ColumnContent, field(default_factory=lambda: ColumnContent("Hub ❤️", "number", False))])40auto_eval_column_dict.append(["still_on_hub", ColumnContent, field(default_factory=lambda: ColumnContent("Available on the hub", "bool", False))])41auto_eval_column_dict.append(["revision", ColumnContent, field(default_factory=lambda: ColumnContent("Model sha", "str", False, False))])42 43# We use make dataclass to dynamically fill the scores from Tasks44AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict)45 46 47## For the queue columns in the submission tab48@dataclass49class EvalQueueColumn:  # Queue column50    model: ColumnContent = field(default_factory=lambda: ColumnContent("model", "markdown", True))51    revision: ColumnContent = field(default_factory=lambda: ColumnContent("revision", "str", True))52    private: ColumnContent = field(default_factory=lambda: ColumnContent("private", "bool", True))53    precision: ColumnContent = field(default_factory=lambda: ColumnContent("precision", "str", True))54    weight_type: ColumnContent = field(default_factory=lambda: ColumnContent("weight_type", "str", "Original"))55    status: ColumnContent = field(default_factory=lambda: ColumnContent("status", "str", True))56 57 58## All the model information that we might need59@dataclass60class ModelDetails:61    name: str62    display_name: str = ""63    symbol: str = ""  # emoji64 65 66class ModelType(Enum):67    PT = ModelDetails(name="pretrained", symbol="🟢")68    FT = ModelDetails(name="fine-tuned", symbol="🔶")69    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")70    RL = ModelDetails(name="RL-tuned", symbol="🟦")71    Unknown = ModelDetails(name="", symbol="?")72 73    def to_str(self, separator=" "):74        return f"{self.value.symbol}{separator}{self.value.name}"75 76    @staticmethod77    def from_str(type):78        if "fine-tuned" in type or "🔶" in type:79            return ModelType.FT80        if "pretrained" in type or "🟢" in type:81            return ModelType.PT82        if "RL-tuned" in type or "🟦" in type:83            return ModelType.RL84        if "instruction-tuned" in type or "⭕" in type:85            return ModelType.IFT86        return ModelType.Unknown87 88 89class WeightType(Enum):90    Adapter = ModelDetails("Adapter")91    Original = ModelDetails("Original")92    Delta = ModelDetails("Delta")93 94 95class Precision(Enum):96    float16 = ModelDetails("float16")97    bfloat16 = ModelDetails("bfloat16")98    Unknown = ModelDetails("?")99 100    def from_str(precision):101        if precision in ["torch.float16", "float16"]:102            return Precision.float16103        if precision in ["torch.bfloat16", "bfloat16"]:104            return Precision.bfloat16105        return Precision.Unknown106 107 108# Column selection109COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]110 111EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]112EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]113 114BENCHMARK_COLS = [t.value.col_name for t in Tasks]115