rhoitjadhav/template-space-docker-v1

1import sys2import time3 4import pandas as pd5import requests6from datasets import load_dataset7 8import argilla as rg9from argilla.labeling.text_classification import Rule, add_rules10 11 12def load_datasets():13    # This is the code that you want to execute when the endpoint is available14    print("Argilla is available! Loading datasets")15    api_key = sys.argv[-1]16    rg.init(api_key=api_key, workspace="team")17 18    # load dataset from json19    my_dataframe = pd.read_json(20        "https://raw.githubusercontent.com/recognai/datasets/main/sst-sentimentclassification.json"21    )22 23    # convert pandas dataframe to DatasetForTextClassification24    dataset_rg = rg.DatasetForTextClassification.from_pandas(my_dataframe)25 26    # Define labeling schema to avoid UI user modification27    settings = rg.TextClassificationSettings(label_schema={"POSITIVE", "NEGATIVE"})28    rg.configure_dataset(name="sst-sentiment-explainability", settings=settings)29 30    # log the dataset31    rg.log(32        dataset_rg,33        name="sst-sentiment-explainability",34        tags={35            "description": "The sst2 sentiment dataset with predictions from a pretrained pipeline and explanations "36            "from Transformers Interpret. "37        },38    )39 40    dataset = load_dataset("argilla/news-summary", split="train").select(range(100))41    dataset_rg = rg.read_datasets(dataset, task="Text2Text")42 43    # log the dataset44    rg.log(45        dataset_rg,46        name="news-text-summarization",47        tags={48            "description": "A text summarization dataset with news pieces and their predicted summaries."49        },50    )51 52    # Read dataset from Hub53    dataset_rg = rg.read_datasets(54        load_dataset("argilla/agnews_weak_labeling", split="train"),55        task="TextClassification",56    )57 58    # Define labeling schema to avoid UI user modification59    settings = rg.TextClassificationSettings(60        label_schema={"World", "Sports", "Sci/Tech", "Business"}61    )62    rg.configure_dataset(name="news-programmatic-labeling", settings=settings)63 64    # log the dataset65    rg.log(66        dataset_rg,67        name="news-programmatic-labeling",68        tags={69            "description": "The AG News with programmatic labeling rules (see weak labeling mode in the UI)."70        },71    )72 73    # define queries and patterns for each category (using ES DSL)74    queries = [75        (["money", "financ*", "dollar*"], "Business"),76        (["war", "gov*", "minister*", "conflict"], "World"),77        (["*ball", "sport*", "game", "play*"], "Sports"),78        (["sci*", "techno*", "computer*", "software", "web"], "Sci/Tech"),79    ]80 81    # define rules82    rules = [83        Rule(query=term, label=label) for terms, label in queries for term in terms84    ]85 86    # add rules to the dataset87    add_rules(dataset="news-programmatic-labeling", rules=rules)88 89    # load dataset from the hub90    dataset = load_dataset("argilla/gutenberg_spacy-ner", split="train")91 92    # read in dataset, assuming it's a dataset for token classification93    dataset_rg = rg.read_datasets(dataset, task="TokenClassification")94 95    # Define labeling schema to avoid UI user modification96    labels = {97        "CARDINAL",98        "DATE",99        "EVENT",100        "FAC",101        "GPE",102        "LANGUAGE",103        "LAW",104        "LOC",105        "MONEY",106        "NORP",107        "ORDINAL",108        "ORG",109        "PERCENT",110        "PERSON",111        "PRODUCT",112        "QUANTITY",113        "TIME",114        "WORK_OF_ART",115    }116    settings = rg.TokenClassificationSettings(label_schema=labels)117    rg.configure_dataset(name="gutenberg_spacy-ner-monitoring", settings=settings)118 119    # log the dataset120    rg.log(121        dataset_rg,122        "gutenberg_spacy-ner-monitoring",123        tags={124            "description": "A dataset containing text from books with predictions from two spaCy NER pre-trained "125            "models. "126        },127    )128 129 130if __name__ == "__main__":131    while True:132        try:133            response = requests.get("http://0.0.0.0:6900/")134            if response.status_code == 200:135                load_datasets()136                break137        except requests.exceptions.ConnectionError:138            pass139        except Exception as e:140            print(e)141            time.sleep(10)142            pass143 144        time.sleep(5)145