CONDA-Workshop/Data-Contamination-Database

1import logging2import re3import os4from concurrent.futures import ThreadPoolExecutor, as_completed5from typing import Dict, List, Union6from urllib.parse import urljoin, urlparse7 8import requests9from bs4 import BeautifulSoup10 11from huggingface_hub import HfApi12 13HF_API = HfApi(token=os.environ.get("TOKEN", None))14 15 16def get_base_url(url: str) -> str:17    """18    Extracts the base URL from a given URL.19 20    Parameters:21    - url (str): The URL to extract the base URL from.22 23    Returns:24    - str: The base URL.25    """26    parsed_url = urlparse(url)27    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"28    return base_url29 30 31def get_domain_name(url: str) -> str:32    """33    Get the domain name from a URL.34 35    Args:36        url (str): The URL.37 38    Returns:39        str: The domain name.40    """41 42    parsed_uri = urlparse(url)43    domain = "{uri.netloc}".format(uri=parsed_uri)44    if domain.startswith("www."):45        domain = domain[4:]46 47    # Remove last domain48    domain = ".".join(domain.split(".")[:-1])49    # First latter in uppercase50    return domain.capitalize()51 52 53def get_favicon(url: str) -> str:54    headers = {55        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"56    }57    try:58        response = requests.get(url, headers=headers, timeout=2)59        if response.status_code == 200:60            soup = BeautifulSoup(response.content, "html.parser")61            # Search for all potential icons including meta tags62            icon_links = soup.find_all(63                "link", rel=re.compile(r"(shortcut icon|icon|apple-touch-icon)", re.I)64            )65            meta_icons = soup.find_all(66                "meta", attrs={"content": re.compile(r".ico$", re.I)}67            )68            icons = icon_links + meta_icons69 70            if icons:71                for icon in icons:72                    favicon_url = icon.get("href") or icon.get("content")73                    if favicon_url:74                        if favicon_url.startswith("/"):75                            favicon_url = urljoin(url, favicon_url)76                        return favicon_url77                # If icons found but no href or content, return default78                return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"79            else:80                # No icons found, return default81                return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"82        else:83            # Response was not OK, return default84            return (85                "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"86            )87    except requests.Timeout:88        logging.warning(f"Request timed out for {url}")89        return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"90    except Exception as e:91        logging.warning(f"An error occurred while fetching favicon for {url}: {e}")92        return "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"93 94 95def download_favicons(urls: List[str]) -> Dict[str, str]:96    favicons = {}97    urls = list(set(urls))98    with ThreadPoolExecutor(max_workers=20) as executor:99        future_to_url = {executor.submit(get_favicon, url): url for url in urls}100        for future in as_completed(future_to_url):101            url = future_to_url[future]102            try:103                favicon_url = future.result()104                favicons[url] = favicon_url105            except Exception as e:106                logging.warning(f"Failed to fetch favicon for {url}: {e}")107                favicons[url] = (108                    "https://upload.wikimedia.org/wikipedia/commons/0/01/Website_icon.svg"109                )110    return favicons111 112 113def url_exists(url):114    """115    Checks if a URL exists by making a HEAD request.116 117    Parameters:118    - url (str): The URL to check.119 120    Returns:121    - bool: True if the URL exists, False otherwise.122    """123    try:124        response = requests.head(url, allow_redirects=True)125        return response.status_code < 400126    except requests.RequestException:127        # In case of network problems, SSL errors, etc.128        return False129 130 131def build_dataset_url(dataset_name: str):132    """133    Build an HTML string with the dataset URL.134    """135    url = f"https://huggingface.co/datasets/{dataset_name}"136    # Test if the url exists137    if url_exists(url) and HF_API.repo_exists(dataset_name, repo_type="dataset"):138        return url139    else:140        return None141 142 143def build_model_url(model_name: str):144    """145    Build an HTML string with the model URL.146    """147    url = f"https://huggingface.co/{model_name}"148    # Test if the url exists149    if url_exists(url) and HF_API.repo_exists(model_name, repo_type="model"):150        return url151    else:152        return None153 154 155def build_text_icon(text: str, url: Union[str, None], icon_url: str):156    if url is not None:157        return (158            f'<a href="{url}" target="_blank" style="text-decoration: none; color: inherit; display: inline-flex; align-items: center;">'159            f'<img src="{icon_url}" alt="{url}" style="display: inline-block; vertical-align: middle; margin-right: 4px;" width="16" height="16">'160            f'<span style="display: inline-block; vertical-align: middle;">{text}</span> </a>'161        )162    else:163        return text164 165 166def build_datasets_urls(datasets_names: List[str]) -> Dict[str, str]:167    """168    Build a dictionary of dataset URLs from a list of dataset names.169 170    Parameters:171    - datasets_names (List[str]): The list of dataset names.172 173    Returns:174    - Dict[str, str]: A dictionary of dataset URLs.175    """176    return {dataset: build_dataset_url(dataset) for dataset in datasets_names}177 178 179def build_models_urls(models_names: List[str]) -> Dict[str, str]:180    """181    Build a dictionary of model URLs from a list of model names.182 183    Parameters:184    - models_names (List[str]): The list of model names.185 186    Returns:187    - Dict[str, str]: A dictionary of model URLs.188    """189    return {model: build_model_url(model) for model in models_names}190