wietsedv/xpos

1import gradio as gr2import gradio.inputs3import gradio.outputs4from transformers.pipelines import pipeline5 6 7lang_names = ['Afrikaans', 'Ancient Greek', 'Arabic', 'Armenian', 'Basque', 'Belarusian', 'Bulgarian', 'Catalan', 'Chinese', 'Classical Chinese', 'Croatian', 'Czech', 'Danish', 'Dutch', 'English', 'Estonian', 'Faroese', 'Finnish', 'French', 'Galician', 'German', 'Gothic', 'Greek', 'Hebrew', 'Hindi', 'Hungarian', 'Icelandic', 'Indonesian', 'Irish', 'Italian', 'Japanese', 'Korean', 'Latin', 'Latvian', 'Lithuanian', 'Maltese', 'Marathi', 'Naija', 'North Sami', 'Norwegian', 'Old Church Slavonic', 'Old East Slavic', 'Old French', 'Persian', 'Polish', 'Portuguese', 'Romanian', 'Russian', 'Sanskrit', 'Scottish Gaelic', 'Serbian', 'Slovak', 'Slovenian', 'Spanish', 'Swedish', 'Tamil', 'Telugu', 'Turkish', 'Ukrainian', 'Urdu', 'Uyghur', 'Vietnamese', 'Welsh', 'Western Armenian', 'Wolof']8 9lang_codes = ['af', 'grc', 'ar', 'hy', 'eu', 'be', 'bg', 'ca', 'zh', 'lzh', 'hr', 'cs', 'da', 'nl', 'en', 'et', 'fo', 'fi', 'fr', 'gl', 'de', 'got', 'el', 'he', 'hi', 'hu', 'is', 'id', 'ga', 'it', 'ja', 'ko', 'la', 'lv', 'lt', 'mt', 'mr', 'pcm', 'sme', 'no', 'cu', 'orv', 'fro', 'fa', 'pl', 'pt', 'ro', 'ru', 'sa', 'gd', 'sr', 'sk', 'sl', 'es', 'sv', 'ta', 'te', 'tr', 'uk', 'ur', 'ug', 'vi', 'cy', 'hyw', 'wo']10 11model_ids = [12    f"wietsedv/xlm-roberta-base-ft-udpos28-{code}" for code in lang_codes13]14 15def model_link(model_id):16    return f"<a href='https://huggingface.co/{model_id}' target='_blank'>🤗 {model_id}</a>"17 18article = "<table style='width:auto'>"19article += "<thead><th>Source language</th><th>Model</th></thead><tbody>"20article += "\n".join([f"<tr><td>{l}</td><td>{model_link(m)}</td></tr>" for l, m in zip(lang_names, model_ids)])21article += "</tbody></table>"22 23loaded_model_id = None24pipe = None25 26def tag(text, lang_index):27    global loaded_model_id, pipe28 29    model_id = model_ids[lang_index]30    if pipe is None or model_id != loaded_model_id:31        loaded_model_id = model_id32        pipe = pipeline("token-classification", model_id, aggregation_strategy="first")33 34    # Aggregate words:35    # split on whitespace and PUNCT, but merge other subtokens (keep first tag)36    out = []37    for g in pipe(text):38        if g["word"][0] == "▁" or g["entity"] == "PUNCT":39            out.append((g["word"].lstrip("▁"), g["entity"]))40        else:41            out[-1] = (out[-1][0] + g["word"], out[-1][1])42 43    return out, model_link(model_id)44 45 46iface = gr.Interface(47    fn=tag,48    inputs=[49        gradio.inputs.Textbox(label="Text", lines=3, placeholder="Enter a sentence here..."),50        gradio.inputs.Dropdown(label="Source language", choices=lang_names, type="index"),51    ],52    outputs=[53        gradio.outputs.HighlightedText(label="Output"),54        gradio.outputs.HTML(label="Model"),55    ],56    title="Cross-lingual part-of-speech tagging",57    description="Enter some text in any language and choose any of 65 source languages. The source language is the language for which XLM-RoBERTa is fine-tuned on Universal Dependencies v2.8 universal part-of-speech tagging data. This space is meant to demonstrate cross-lingual transfer, so the language of your sentence and the selected language do not have to match. You may find fewer mistakes if the selected language is similar to the actual language of your text.",58    allow_screenshot=False,59    allow_flagging="never",60    article=article,61    theme="huggingface",62    # examples=[["Dit is een test.", "English"]]63)64iface.launch()65