build-small-hackathon/hackathon-advisor

1from __future__ import annotations2 3from collections import Counter, defaultdict4from collections.abc import Mapping, Sequence5import math6from typing import Any7 8from hackathon_advisor.data import (9    Project,10    ProjectIndex,11    normalize_project_tags,12    public_project_summary,13    public_project_title,14    tokenize,15)16from hackathon_advisor.quest_taxonomy import QUESTS, normalize_match, quest_profiles17from hackathon_advisor._text import utc_now18 19 20DASHBOARD_SCHEMA_VERSION = 121TSNE_RANDOM_STATE = 4222TSNE_MIN_PROJECTS = 323LINKS_PER_PROJECT = 224CLUSTER_LABEL_ALGORITHM = "distinctive-keywords-v1"25 26STOPWORDS = {27    "about",28    "agent",29    "app",30    "apps",31    "ai",32    "all",33    "an",34    "and",35    "are",36    "as",37    "at",38    "before",39    "assistant",40    "be",41    "been",42    "being",43    "build",44    "build-small",45    "build-small-hackathon",46    "built",47    "by",48    "demo",49    "face",50    "for",51    "from",52    "gradio",53    "hackathon",54    "hugging",55    "huggingface",56    "in",57    "is",58    "it",59    "its",60    "first",61    "local",62    "make",63    "makes",64    "made",65    "me",66    "model",67    "models",68    "my",69    "of",70    "on",71    "or",72    "our",73    "one",74    "project",75    "projects",76    "pro",77    "region",78    "run",79    "runs",80    "small",81    "space",82    "spaces",83    "submission",84    "the",85    "their",86    "them",87    "these",88    "they",89    "this",90    "those",91    "to",92    "tool",93    "tools",94    "try",95    "us",96    "use",97    "used",98    "uses",99    "using",100    "we",101    "with",102    "you",103    "your",104}105 106 107class DashboardError(ValueError):108    pass109 110 111def build_dashboard_payload(112    index: ProjectIndex,113    *,114    quest_matches: Mapping[str, Sequence[Mapping[str, Any]]] | None = None,115    quest_source: str = "",116    generated_at: str | None = None,117) -> dict[str, Any]:118    projects = list(index.projects)119    if len(projects) < TSNE_MIN_PROJECTS:120        raise DashboardError(f"dashboard atlas requires at least {TSNE_MIN_PROJECTS} projects")121 122    matrix = _embedding_matrix(index)123    coordinates = _tsne_coordinates(matrix)124    raw_cluster_labels = _cluster_labels(matrix)125    cluster_id_by_raw, clusters = _cluster_payloads(projects, coordinates, raw_cluster_labels)126    normalized_quest_matches = _normalize_quest_matches(projects, quest_matches)127    points = _point_payloads(projects, coordinates, raw_cluster_labels, cluster_id_by_raw, normalized_quest_matches)128    links = _nearest_links(projects, matrix)129    quest_report = _quest_report(points, normalized_quest_matches, quest_source)130    payload = {131        "schema_version": DASHBOARD_SCHEMA_VERSION,132        "generated_at": generated_at or utc_now(),133        "project_count": len(projects),134        "provenance": {135            "snapshot_generated_at": index.generated_at,136            "snapshot_source": index.source,137            "index_generated_at": index.index_generated_at,138            "index_algorithm": index.index_algorithm,139            "snapshot_digest": index.snapshot_digest,140            "embedding": index.embedding_metadata,141        },142        "layout": {143            "algorithm": "tsne",144            "metric": "cosine",145            "init": "pca",146            "random_state": TSNE_RANDOM_STATE,147            "perplexity": _tsne_perplexity(len(projects)),148        },149        "cluster_label_algorithm": CLUSTER_LABEL_ALGORITHM,150        "points": points,151        "links": links,152        "clusters": clusters,153        "quest_report": quest_report,154    }155    validate_dashboard_payload(payload)156    return payload157 158 159def validate_dashboard_payload(payload: Mapping[str, Any]) -> None:160    if payload.get("schema_version") != DASHBOARD_SCHEMA_VERSION:161        raise DashboardError("unsupported dashboard schema version")162    project_count = int(payload.get("project_count") or 0)163    if project_count < TSNE_MIN_PROJECTS:164        raise DashboardError("dashboard project count is too small")165    points = payload.get("points")166    if not isinstance(points, list) or len(points) != project_count:167        raise DashboardError("dashboard point count does not match project count")168    ids: set[str] = set()169    cluster_ids: set[str] = set()170    for point in points:171        if not isinstance(point, dict):172            raise DashboardError("dashboard points must be objects")173        project_id = str(point.get("id") or "")174        if not project_id or project_id in ids:175            raise DashboardError("dashboard points must have unique project ids")176        ids.add(project_id)177        x = float(point.get("x"))178        y = float(point.get("y"))179        if not 0.0 <= x <= 100.0 or not 0.0 <= y <= 100.0:180            raise DashboardError("dashboard point coordinates must be percentages")181        cluster_id = str(point.get("cluster_id") or "")182        if not cluster_id:183            raise DashboardError("dashboard point cluster id is missing")184        cluster_ids.add(cluster_id)185 186    clusters = payload.get("clusters")187    if not isinstance(clusters, list) or not clusters:188        raise DashboardError("dashboard clusters are missing")189    declared_cluster_ids = {str(cluster.get("id") or "") for cluster in clusters if isinstance(cluster, dict)}190    if cluster_ids - declared_cluster_ids:191        raise DashboardError("dashboard points reference missing clusters")192 193    links = payload.get("links")194    if not isinstance(links, list):195        raise DashboardError("dashboard links must be a list")196    for link in links:197        if str(link.get("source") or "") not in ids or str(link.get("target") or "") not in ids:198            raise DashboardError("dashboard link references an unknown project")199 200    quest_report = payload.get("quest_report")201    if not isinstance(quest_report, dict):202        raise DashboardError("dashboard quest report is missing")203    if quest_report.get("status") not in {"analyzed", "not_analyzed"}:204        raise DashboardError("dashboard quest report status is invalid")205 206 207def _embedding_matrix(index: ProjectIndex) -> Any:208    import numpy as np209 210    return np.asarray(index.project_vectors(), dtype=np.float32)211 212 213def _tsne_coordinates(matrix: Any) -> list[tuple[float, float]]:214    from sklearn.manifold import TSNE215 216    coords = TSNE(217        n_components=2,218        perplexity=_tsne_perplexity(int(matrix.shape[0])),219        init="pca",220        learning_rate="auto",221        max_iter=1000,222        metric="cosine",223        random_state=TSNE_RANDOM_STATE,224    ).fit_transform(matrix)225    return _scale_points(coords)226 227 228def _tsne_perplexity(count: int) -> int:229    return max(2, min(30, count // 4))230 231 232def _cluster_labels(matrix: Any) -> list[int]:233    from sklearn.cluster import KMeans234 235    count = int(matrix.shape[0])236    cluster_count = min(10, max(min(6, count), round(math.sqrt(count))))237    labels = KMeans(238        n_clusters=cluster_count,239        random_state=TSNE_RANDOM_STATE,240        n_init=20,241    ).fit_predict(matrix)242    return [int(label) for label in labels]243 244 245def _scale_points(points: Any, low: float = 3.0, high: float = 97.0) -> list[tuple[float, float]]:246    import numpy as np247 248    scaled = np.empty_like(points, dtype=np.float64)249    for axis in range(points.shape[1]):250        column = points[:, axis]251        minimum = float(column.min())252        maximum = float(column.max())253        span = maximum - minimum254        if span <= 1e-9:255            scaled[:, axis] = (low + high) / 2.0256        else:257            scaled[:, axis] = low + (column - minimum) / span * (high - low)258    return [(round(float(x), 4), round(float(y), 4)) for x, y in scaled]259 260 261def _cluster_payloads(262    projects: Sequence[Project],263    coordinates: Sequence[tuple[float, float]],264    raw_labels: Sequence[int],265) -> tuple[dict[int, str], list[dict[str, Any]]]:266    grouped: dict[int, list[int]] = defaultdict(list)267    for index, label in enumerate(raw_labels):268        grouped[int(label)].append(index)269 270    ordered_raw_labels = sorted(271        grouped,272        key=lambda label: (-len(grouped[label]), _cluster_center(coordinates, grouped[label])),273    )274    cluster_id_by_raw = {label: f"cluster-{position + 1}" for position, label in enumerate(ordered_raw_labels)}275    clusters: list[dict[str, Any]] = []276    corpus_document_frequency = _corpus_document_frequency(projects)277    for raw_label in ordered_raw_labels:278        indexes = grouped[raw_label]279        cluster_projects = [projects[index] for index in indexes]280        representatives = sorted(281            cluster_projects,282            key=lambda project: (project.likes, project.last_modified, project.title.lower()),283            reverse=True,284        )[:4]285        keywords = _cluster_keywords(286            cluster_projects,287            corpus_document_frequency=corpus_document_frequency,288            corpus_project_count=len(projects),289        )290        label = (291            " / ".join(word.title() for word in keywords[:2])292            if keywords293            else _representative_cluster_label(representatives)294        )295        clusters.append(296            {297                "id": cluster_id_by_raw[raw_label],298                "label": label,299                "keywords": keywords,300                "project_count": len(indexes),301                "center": {302                    "x": round(sum(coordinates[index][0] for index in indexes) / len(indexes), 4),303                    "y": round(sum(coordinates[index][1] for index in indexes) / len(indexes), 4),304                },305                "representative_projects": [project.to_public_dict() for project in representatives],306            }307        )308    return cluster_id_by_raw, clusters309 310 311def _cluster_center(coordinates: Sequence[tuple[float, float]], indexes: Sequence[int]) -> tuple[float, float]:312    return (313        sum(coordinates[index][0] for index in indexes) / len(indexes),314        sum(coordinates[index][1] for index in indexes) / len(indexes),315    )316 317 318def _corpus_document_frequency(projects: Sequence[Project]) -> Counter[str]:319    document_frequency: Counter[str] = Counter()320    for project in projects:321        document_frequency.update(set(_project_keyword_tokens(project)))322    return document_frequency323 324 325def _cluster_keywords(326    projects: Sequence[Project],327    *,328    corpus_document_frequency: Mapping[str, int],329    corpus_project_count: int,330) -> list[str]:331    counts: Counter[str] = Counter()332    document_frequency: Counter[str] = Counter()333    project_list = list(projects)334    for project in project_list:335        tokens = _project_keyword_tokens(project)336        counts.update(tokens)337        document_frequency.update(set(tokens))338 339    if not project_list:340        return []341 342    min_cluster_documents = 1 if len(project_list) <= 3 else 2343    scored: list[tuple[float, int, int, str]] = []344    for token, count in counts.items():345        cluster_documents = document_frequency[token]346        if cluster_documents < min_cluster_documents:347            continue348        corpus_documents = int(corpus_document_frequency.get(token) or 0)349        if corpus_documents <= 0:350            continue351        inverse_document_frequency = math.log((1 + corpus_project_count) / (1 + corpus_documents))352        if inverse_document_frequency <= 0.0:353            continue354        exclusivity = cluster_documents / corpus_documents355        coverage = cluster_documents / len(project_list)356        score = (357            (1.0 + math.log(count))358            * inverse_document_frequency359            * (0.35 + 0.65 * exclusivity)360            * (0.35 + 0.65 * coverage)361        )362        scored.append((score, cluster_documents, count, token))363 364    scored.sort(key=lambda item: (-item[0], -item[1], -item[2], item[3]))365    return [token for _score, _cluster_documents, _count, token in scored[:5]]366 367 368def _project_keyword_tokens(project: Project) -> list[str]:369    text = " ".join(370        [371            project.title,372            project.slug.replace("-", " ").replace("_", " "),373            project.summary,374            " ".join(normalize_project_tags(project.tags)),375            " ".join(project.models),376        ]377    )378    return [token for token in tokenize(text) if _is_cluster_keyword(token)]379 380 381def _is_cluster_keyword(token: str) -> bool:382    if token in STOPWORDS:383        return False384    if token.startswith("region"):385        return False386    if token.isdigit():387        return False388    return True389 390 391def _representative_cluster_label(projects: Sequence[Project]) -> str:392    labels: list[str] = []393    for project in projects:394        title = public_project_title(project.title)395        if title == "Untitled project":396            continue397        labels.append(title)398        if len(labels) == 2:399            break400    return " / ".join(labels) if labels else "Mixed projects"401 402 403def _normalize_quest_matches(404    projects: Sequence[Project],405    quest_matches: Mapping[str, Sequence[Mapping[str, Any]]] | None,406) -> dict[str, list[dict[str, Any]]]:407    project_ids = {project.id for project in projects}408    normalized = {project.id: [] for project in projects}409    if quest_matches is None:410        return normalized411    if set(quest_matches) != project_ids:412        missing = sorted(project_ids - set(quest_matches))413        extra = sorted(set(quest_matches) - project_ids)414        detail = []415        if missing:416            detail.append(f"missing {len(missing)} projects")417        if extra:418            detail.append(f"unknown {len(extra)} projects")419        raise DashboardError("quest analysis project coverage is invalid: " + ", ".join(detail))420    for project_id, matches in quest_matches.items():421        normalized[project_id] = [_normalize_quest_match(match) for match in matches]422    return normalized423 424 425def _normalize_quest_match(match: Mapping[str, Any]) -> dict[str, Any]:426    try:427        return normalize_match(match)428    except ValueError as error:429        raise DashboardError(f"invalid quest match: {error}") from error430 431 432def _point_payloads(433    projects: Sequence[Project],434    coordinates: Sequence[tuple[float, float]],435    raw_labels: Sequence[int],436    cluster_id_by_raw: Mapping[int, str],437    quest_matches: Mapping[str, Sequence[Mapping[str, Any]]],438) -> list[dict[str, Any]]:439    points: list[dict[str, Any]] = []440    for project, (x, y), raw_label in zip(projects, coordinates, raw_labels, strict=True):441        matches = list(quest_matches.get(project.id) or [])442        points.append(443            {444                "id": project.id,445                "title": public_project_title(project.title),446                "summary": public_project_summary(project.summary),447                "url": project.url,448                "host": project.host,449                "likes": project.likes,450                "sdk": project.sdk,451                "models": list(project.models),452                "tags": list(normalize_project_tags(project.tags)),453                "last_modified": project.last_modified,454                "x": x,455                "y": y,456                "cluster_id": cluster_id_by_raw[int(raw_label)],457                "quest_matches": matches,458                "quest_ids": [str(match["quest"]) for match in matches],459            }460        )461    return points462 463 464def _nearest_links(projects: Sequence[Project], matrix: Any) -> list[dict[str, Any]]:465    import numpy as np466 467    similarity = matrix @ matrix.T468    pairs: dict[tuple[int, int], float] = {}469    for index in range(len(projects)):470        order = np.argsort(similarity[index])[::-1]471        neighbors = [int(candidate) for candidate in order if int(candidate) != index][:LINKS_PER_PROJECT]472        for neighbor in neighbors:473            left, right = sorted((index, neighbor))474            pairs[(left, right)] = max(float(similarity[left, right]), pairs.get((left, right), -1.0))475    return [476        {477            "source": projects[left].id,478            "target": projects[right].id,479            "score": round(max(0.0, min(1.0, score)), 4),480        }481        for (left, right), score in sorted(pairs.items(), key=lambda item: (-item[1], item[0]))482    ]483 484 485def _quest_report(486    points: Sequence[Mapping[str, Any]],487    quest_matches: Mapping[str, Sequence[Mapping[str, Any]]],488    quest_source: str,489) -> dict[str, Any]:490    profiles = {profile["id"]: profile for profile in quest_profiles()}491    status = "analyzed" if quest_source else "not_analyzed"492    quests = []493    for quest in QUESTS:494        matched_points = [495            point496            for point in points497            if any(match["quest"] == quest for match in quest_matches.get(str(point["id"]), []))498        ]499        examples = sorted(500            matched_points,501            key=lambda point: (502                max(503                    (504                        float(match["confidence"])505                        for match in quest_matches.get(str(point["id"]), [])506                        if match["quest"] == quest507                    ),508                    default=0.0,509                ),510                int(point.get("likes") or 0),511            ),512            reverse=True,513        )[:4]514        profile = profiles.get(quest, {"label": quest, "description": ""})515        quests.append(516            {517                "id": quest,518                "label": profile["label"],519                "description": profile["description"],520                "project_count": len(matched_points),521                "examples": [522                    {523                        "id": point["id"],524                        "title": point["title"],525                        "url": point["url"],526                    }527                    for point in examples528                ],529            }530        )531    return {532        "status": status,533        "source": quest_source,534        "quests": quests,535    }536