language-tools/language-demo

1import os2 3import fasttext4import gradio as gr5from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline6import torch7 8title = "Community Tab Language Detection & Translation"9description = """10When comments are created in the community tab, detect the language of the content.11Then, if the detected language is different from the user's language, display an option to translate it.12"""13 14model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M")15tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")16device = 0 if torch.cuda.is_available() else -117print(f"Is CUDA available: {torch.cuda.is_available()}")18 19language_code_map = {20    "English": "eng_Latn",21    "French": "fra_Latn",22    "German": "deu_Latn",23    "Spanish": "spa_Latn",24    "Korean": "kor_Hang",25    "Japanese": "jpn_Jpan"26}27 28 29def identify_language(text):30    model_file = "lid218e.bin"31    model_full_path = os.path.join(os.path.dirname(__file__), model_file)32    model = fasttext.load_model(model_full_path)33    predictions = model.predict(text, k=1)  # e.g., (('__label__eng_Latn',), array([0.81148803]))34 35    CHAR_TO_STRIP = 9  # To strip away '__label__' from language code36    language_code = predictions[0][0][CHAR_TO_STRIP:]37 38    return language_code39 40 41def display(user_lang, text):42    user_lang_code = language_code_map[user_lang]43    language_code = identify_language(text)44 45    translate_button_visibility = language_code != user_lang_code46 47    detected_language_text = f"""48    Detected Language: {language_code}\n49    User Content Language: {user_lang_code}\n50    {"" if translate_button_visibility else "[NOT TRANSLATABLE] Detected Language and Content Language are the same"}51    """52 53    return text, gr.update(value="", placeholder="Leave a comment"), gr.update(value=detected_language_text), gr.update(visible=translate_button_visibility, variant="primary")54 55 56def translate(text, src_lang, tgt_lang):57    CHAR_TO_STRIP = 22  # To strip away 'Detected Language: ' from language code58    LANGUAGE_CODE_LENGTH = 8  # To strip away 'Detected Language: ' from language code59    src_lang_code = src_lang[CHAR_TO_STRIP:CHAR_TO_STRIP + LANGUAGE_CODE_LENGTH]60    tgt_lang_code = language_code_map[tgt_lang]61 62    translation_pipeline = pipeline(63        "translation", model=model, tokenizer=tokenizer, src_lang=src_lang_code, tgt_lang=tgt_lang_code, device=device)64    result = translation_pipeline(text)65    return result[0]['translation_text']66 67with gr.Blocks() as demo:68    gr.HTML(69        f"""70            <div style="text-align: center; margin: 0 auto;">71              <div style=" display: inline-flex; align-items: center; gap: 0.8rem; font-size: 1.75rem;">72                <h1 style="font-weight: 900; margin-bottom: 7px;margin-top:5px">73                  {title}74                </h1>75              </div>76              <p style="margin-bottom: 10px; font-size: 94%; line-height: 23px;">77                {description}78              </p>79            </div>80        """81    )82 83    user_langugage_radio = gr.Radio(["English", "Spanish", "Korean", "French", "German", "Japanese"],84                                    value="English", label="User Content Language")85 86    comment_input_textbox = gr.Textbox(87        lines=3, label="Write a Comment", placeholder="Leave a comment")88    comment_out_textbox = gr.Textbox(lines=3, label="Comment")89    detected_lang_markdown = gr.Markdown("", elem_id="detect-lang-md")90 91    comment_btn = gr.Button("Comment")92 93    translate_btn = gr.Button("Translate", visible=False)94    detected_language_value = gr.Textbox("", visible=False)95 96 97    comment_btn.click(display,98                      inputs=[user_langugage_radio, comment_input_textbox],99                      outputs=[100                          comment_out_textbox,101                          comment_input_textbox,102                          detected_lang_markdown,103                          translate_btn104                      ])105 106    translate_btn.click(translate,107                        inputs=[108                            comment_out_textbox,109                            detected_lang_markdown,110                            user_langugage_radio111                        ],112                        outputs=comment_out_textbox)113 114demo.launch()115