Datasetpublic

tengomucho/english_quotes_sanitized

This dataset is the same as Abirate/english_quotes, but I sanitized the author and sanitized the text to avoid weird characters. from ftfy import fix_encoding from datasets import load_dataset def correct_encoding(examples): quote = examples["quote"] author = examples["author"] # remove trailing comma from authors and fix encoding author = author.rstrip(",") author = fix_encoding(author) examples["author"] = author # fix encoding quote = fix_encoding(quote)… See the full description on the dataset page: https://huggingface.co/datasets/tengomucho/english_quotes_sanitized.

sourceHugging Faceupdated 1y agoView on Hugging Face

0likes33downloads

Dataset Card

This dataset is the same as Abirate/english_quotes, but I sanitized the author and sanitized the text to avoid weird characters.

python

from ftfy import fix_encoding
from datasets import load_dataset


def correct_encoding(examples):
    quote = examples["quote"]
    author = examples["author"]

    # remove trailing comma from authors and fix encoding
    author = author.rstrip(",")
    author = fix_encoding(author)
    examples["author"] = author

    # fix encoding
    quote = fix_encoding(quote)
    examples["quote"] = quote

    return examples


def fix_english_quotes_dataset():
    dataset_id = "Abirate/english_quotes"
    dataset = load_dataset(dataset_id)
    dataset = dataset.map(correct_encoding)

    # push to hub
    dataset.push_to_hub("tengomucho/english_quotes_sanitized")


if __name__ == "__main__":
    fix_english_quotes_dataset()