mbarnig/ACE-Step
0
1import os2import re3import textwrap4from functools import cached_property5 6import pypinyin7import torch8from hangul_romanize import Transliter9from hangul_romanize.rule import academic10from num2words import num2words11from spacy.lang.ar import Arabic12from spacy.lang.en import English13from spacy.lang.es import Spanish14from spacy.lang.ja import Japanese15from spacy.lang.zh import Chinese16from tokenizers import Tokenizer17 18from .zh_num2words import TextNorm as zh_num2words19from typing import Dict, List, Optional, Set, Union20 21 22#copy from https://github.com/coqui-ai/TTS/blob/dbf1a08a0d4e47fdad6172e433eeb34bc6b13b4e/TTS/tts/layers/xtts/tokenizer.py23def get_spacy_lang(lang):24 if lang == "zh":25 return Chinese()26 elif lang == "ja":27 return Japanese()28 elif lang == "ar":29 return Arabic()30 elif lang == "es":31 return Spanish()32 else:33 # For most languages, Enlish does the job34 return English()35 36 37def split_sentence(text, lang, text_split_length=250):38 """Preprocess the input text"""39 text_splits = []40 if text_split_length is not None and len(text) >= text_split_length:41 text_splits.append("")42 nlp = get_spacy_lang(lang)43 nlp.add_pipe("sentencizer")44 doc = nlp(text)45 for sentence in doc.sents:46 if len(text_splits[-1]) + len(str(sentence)) <= text_split_length:47 # if the last sentence + the current sentence is less than the text_split_length48 # then add the current sentence to the last sentence49 text_splits[-1] += " " + str(sentence)50 text_splits[-1] = text_splits[-1].lstrip()51 elif len(str(sentence)) > text_split_length:52 # if the current sentence is greater than the text_split_length53 for line in textwrap.wrap(54 str(sentence),55 width=text_split_length,56 drop_whitespace=True,57 break_on_hyphens=False,58 tabsize=1,59 ):60 text_splits.append(str(line))61 else:62 text_splits.append(str(sentence))63 64 if len(text_splits) > 1:65 if text_splits[0] == "":66 del text_splits[0]67 else:68 text_splits = [text.lstrip()]69 70 return text_splits71 72 73_whitespace_re = re.compile(r"\s+")74 75# List of (regular expression, replacement) pairs for abbreviations:76_abbreviations = {77 "en": [78 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])79 for x in [80 ("mrs", "misess"),81 ("mr", "mister"),82 ("dr", "doctor"),83 ("st", "saint"),84 ("co", "company"),85 ("jr", "junior"),86 ("maj", "major"),87 ("gen", "general"),88 ("drs", "doctors"),89 ("rev", "reverend"),90 ("lt", "lieutenant"),91 ("hon", "honorable"),92 ("sgt", "sergeant"),93 ("capt", "captain"),94 ("esq", "esquire"),95 ("ltd", "limited"),96 ("col", "colonel"),97 ("ft", "fort"),98 ]99 ],100 "es": [101 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])102 for x in [103 ("sra", "señora"),104 ("sr", "señor"),105 ("dr", "doctor"),106 ("dra", "doctora"),107 ("st", "santo"),108 ("co", "compañía"),109 ("jr", "junior"),110 ("ltd", "limitada"),111 ]112 ],113 "fr": [114 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])115 for x in [116 ("mme", "madame"),117 ("mr", "monsieur"),118 ("dr", "docteur"),119 ("st", "saint"),120 ("co", "compagnie"),121 ("jr", "junior"),122 ("ltd", "limitée"),123 ]124 ],125 "de": [126 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])127 for x in [128 ("fr", "frau"),129 ("dr", "doktor"),130 ("st", "sankt"),131 ("co", "firma"),132 ("jr", "junior"),133 ]134 ],135 "pt": [136 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])137 for x in [138 ("sra", "senhora"),139 ("sr", "senhor"),140 ("dr", "doutor"),141 ("dra", "doutora"),142 ("st", "santo"),143 ("co", "companhia"),144 ("jr", "júnior"),145 ("ltd", "limitada"),146 ]147 ],148 "it": [149 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])150 for x in [151 # ("sig.ra", "signora"),152 ("sig", "signore"),153 ("dr", "dottore"),154 ("st", "santo"),155 ("co", "compagnia"),156 ("jr", "junior"),157 ("ltd", "limitata"),158 ]159 ],160 "pl": [161 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])162 for x in [163 ("p", "pani"),164 ("m", "pan"),165 ("dr", "doktor"),166 ("sw", "święty"),167 ("jr", "junior"),168 ]169 ],170 "ar": [171 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])172 for x in [173 # There are not many common abbreviations in Arabic as in English.174 ]175 ],176 "zh": [177 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])178 for x in [179 # Chinese doesn't typically use abbreviations in the same way as Latin-based scripts.180 ]181 ],182 "cs": [183 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])184 for x in [185 ("dr", "doktor"), # doctor186 ("ing", "inženýr"), # engineer187 ("p", "pan"), # Could also map to pani for woman but no easy way to do it188 # Other abbreviations would be specialized and not as common.189 ]190 ],191 "ru": [192 (re.compile("\\b%s\\b" % x[0], re.IGNORECASE), x[1])193 for x in [194 ("г-жа", "госпожа"), # Mrs.195 ("г-н", "господин"), # Mr.196 ("д-р", "доктор"), # doctor197 # Other abbreviations are less common or specialized.198 ]199 ],200 "nl": [201 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])202 for x in [203 ("dhr", "de heer"), # Mr.204 ("mevr", "mevrouw"), # Mrs.205 ("dr", "dokter"), # doctor206 ("jhr", "jonkheer"), # young lord or nobleman207 # Dutch uses more abbreviations, but these are the most common ones.208 ]209 ],210 "tr": [211 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])212 for x in [213 ("b", "bay"), # Mr.214 ("byk", "büyük"), # büyük215 ("dr", "doktor"), # doctor216 # Add other Turkish abbreviations here if needed.217 ]218 ],219 "hu": [220 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])221 for x in [222 ("dr", "doktor"), # doctor223 ("b", "bácsi"), # Mr.224 ("nőv", "nővér"), # nurse225 # Add other Hungarian abbreviations here if needed.226 ]227 ],228 "ko": [229 (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])230 for x in [231 # Korean doesn't typically use abbreviations in the same way as Latin-based scripts.232 ]233 ],234}235 236 237def expand_abbreviations_multilingual(text, lang="en"):238 for regex, replacement in _abbreviations[lang]:239 text = re.sub(regex, replacement, text)240 return text241 242 243_symbols_multilingual = {244 "en": [245 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])246 for x in [247 ("&", " and "),248 ("@", " at "),249 ("%", " percent "),250 ("#", " hash "),251 ("$", " dollar "),252 ("£", " pound "),253 ("°", " degree "),254 ]255 ],256 "es": [257 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])258 for x in [259 ("&", " y "),260 ("@", " arroba "),261 ("%", " por ciento "),262 ("#", " numeral "),263 ("$", " dolar "),264 ("£", " libra "),265 ("°", " grados "),266 ]267 ],268 "fr": [269 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])270 for x in [271 ("&", " et "),272 ("@", " arobase "),273 ("%", " pour cent "),274 ("#", " dièse "),275 ("$", " dollar "),276 ("£", " livre "),277 ("°", " degrés "),278 ]279 ],280 "de": [281 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])282 for x in [283 ("&", " und "),284 ("@", " at "),285 ("%", " prozent "),286 ("#", " raute "),287 ("$", " dollar "),288 ("£", " pfund "),289 ("°", " grad "),290 ]291 ],292 "pt": [293 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])294 for x in [295 ("&", " e "),296 ("@", " arroba "),297 ("%", " por cento "),298 ("#", " cardinal "),299 ("$", " dólar "),300 ("£", " libra "),301 ("°", " graus "),302 ]303 ],304 "it": [305 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])306 for x in [307 ("&", " e "),308 ("@", " chiocciola "),309 ("%", " per cento "),310 ("#", " cancelletto "),311 ("$", " dollaro "),312 ("£", " sterlina "),313 ("°", " gradi "),314 ]315 ],316 "pl": [317 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])318 for x in [319 ("&", " i "),320 ("@", " małpa "),321 ("%", " procent "),322 ("#", " krzyżyk "),323 ("$", " dolar "),324 ("£", " funt "),325 ("°", " stopnie "),326 ]327 ],328 "ar": [329 # Arabic330 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])331 for x in [332 ("&", " و "),333 ("@", " على "),334 ("%", " في المئة "),335 ("#", " رقم "),336 ("$", " دولار "),337 ("£", " جنيه "),338 ("°", " درجة "),339 ]340 ],341 "zh": [342 # Chinese343 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])344 for x in [345 ("&", " 和 "),346 ("@", " 在 "),347 ("%", " 百分之 "),348 ("#", " 号 "),349 ("$", " 美元 "),350 ("£", " 英镑 "),351 ("°", " 度 "),352 ]353 ],354 "cs": [355 # Czech356 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])357 for x in [358 ("&", " a "),359 ("@", " na "),360 ("%", " procento "),361 ("#", " křížek "),362 ("$", " dolar "),363 ("£", " libra "),364 ("°", " stupně "),365 ]366 ],367 "ru": [368 # Russian369 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])370 for x in [371 ("&", " и "),372 ("@", " собака "),373 ("%", " процентов "),374 ("#", " номер "),375 ("$", " доллар "),376 ("£", " фунт "),377 ("°", " градус "),378 ]379 ],380 "nl": [381 # Dutch382 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])383 for x in [384 ("&", " en "),385 ("@", " bij "),386 ("%", " procent "),387 ("#", " hekje "),388 ("$", " dollar "),389 ("£", " pond "),390 ("°", " graden "),391 ]392 ],393 "tr": [394 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])395 for x in [396 ("&", " ve "),397 ("@", " at "),398 ("%", " yüzde "),399 ("#", " diyez "),400 ("$", " dolar "),401 ("£", " sterlin "),402 ("°", " derece "),403 ]404 ],405 "hu": [406 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])407 for x in [408 ("&", " és "),409 ("@", " kukac "),410 ("%", " százalék "),411 ("#", " kettőskereszt "),412 ("$", " dollár "),413 ("£", " font "),414 ("°", " fok "),415 ]416 ],417 "ko": [418 # Korean419 (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])420 for x in [421 ("&", " 그리고 "),422 ("@", " 에 "),423 ("%", " 퍼센트 "),424 ("#", " 번호 "),425 ("$", " 달러 "),426 ("£", " 파운드 "),427 ("°", " 도 "),428 ]429 ],430}431 432 433def expand_symbols_multilingual(text, lang="en"):434 for regex, replacement in _symbols_multilingual[lang]:435 text = re.sub(regex, replacement, text)436 text = text.replace(" ", " ") # Ensure there are no double spaces437 return text.strip()438 439 440_ordinal_re = {441 "en": re.compile(r"([0-9]+)(st|nd|rd|th)"),442 "es": re.compile(r"([0-9]+)(º|ª|er|o|a|os|as)"),443 "fr": re.compile(r"([0-9]+)(º|ª|er|re|e|ème)"),444 "de": re.compile(r"([0-9]+)(st|nd|rd|th|º|ª|\.(?=\s|$))"),445 "pt": re.compile(r"([0-9]+)(º|ª|o|a|os|as)"),446 "it": re.compile(r"([0-9]+)(º|°|ª|o|a|i|e)"),447 "pl": re.compile(r"([0-9]+)(º|ª|st|nd|rd|th)"),448 "ar": re.compile(r"([0-9]+)(ون|ين|ث|ر|ى)"),449 "cs": re.compile(r"([0-9]+)\.(?=\s|$)"), # In Czech, a dot is often used after the number to indicate ordinals.450 "ru": re.compile(r"([0-9]+)(-й|-я|-е|-ое|-ье|-го)"),451 "nl": re.compile(r"([0-9]+)(de|ste|e)"),452 "tr": re.compile(r"([0-9]+)(\.|inci|nci|uncu|üncü|\.)"),453 "hu": re.compile(r"([0-9]+)(\.|adik|edik|odik|edik|ödik|ödike|ik)"),454 "ko": re.compile(r"([0-9]+)(번째|번|차|째)"),455}456_number_re = re.compile(r"[0-9]+")457_currency_re = {458 "USD": re.compile(r"((\$[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+\$))"),459 "GBP": re.compile(r"((£[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+£))"),460 "EUR": re.compile(r"(([0-9\.\,]*[0-9]+€)|((€[0-9\.\,]*[0-9]+)))"),461}462 463_comma_number_re = re.compile(r"\b\d{1,3}(,\d{3})*(\.\d+)?\b")464_dot_number_re = re.compile(r"\b\d{1,3}(.\d{3})*(\,\d+)?\b")465_decimal_number_re = re.compile(r"([0-9]+[.,][0-9]+)")466 467 468def _remove_commas(m):469 text = m.group(0)470 if "," in text:471 text = text.replace(",", "")472 return text473 474 475def _remove_dots(m):476 text = m.group(0)477 if "." in text:478 text = text.replace(".", "")479 return text480 481 482def _expand_decimal_point(m, lang="en"):483 amount = m.group(1).replace(",", ".")484 return num2words(float(amount), lang=lang if lang != "cs" else "cz")485 486 487def _expand_currency(m, lang="en", currency="USD"):488 amount = float((re.sub(r"[^\d.]", "", m.group(0).replace(",", "."))))489 full_amount = num2words(amount, to="currency", currency=currency, lang=lang if lang != "cs" else "cz")490 491 and_equivalents = {492 "en": ", ",493 "es": " con ",494 "fr": " et ",495 "de": " und ",496 "pt": " e ",497 "it": " e ",498 "pl": ", ",499 "cs": ", ",500 "ru": ", ",501 "nl": ", ",502 "ar": ", ",503 "tr": ", ",504 "hu": ", ",505 "ko": ", ",506 }507 508 if amount.is_integer():509 last_and = full_amount.rfind(and_equivalents[lang])510 if last_and != -1:511 full_amount = full_amount[:last_and]512 513 return full_amount514 515 516def _expand_ordinal(m, lang="en"):517 return num2words(int(m.group(1)), ordinal=True, lang=lang if lang != "cs" else "cz")518 519 520def _expand_number(m, lang="en"):521 return num2words(int(m.group(0)), lang=lang if lang != "cs" else "cz")522 523 524def expand_numbers_multilingual(text, lang="en"):525 if lang == "zh":526 text = zh_num2words()(text)527 else:528 if lang in ["en", "ru"]:529 text = re.sub(_comma_number_re, _remove_commas, text)530 else:531 text = re.sub(_dot_number_re, _remove_dots, text)532 try:533 text = re.sub(_currency_re["GBP"], lambda m: _expand_currency(m, lang, "GBP"), text)534 text = re.sub(_currency_re["USD"], lambda m: _expand_currency(m, lang, "USD"), text)535 text = re.sub(_currency_re["EUR"], lambda m: _expand_currency(m, lang, "EUR"), text)536 except:537 pass538 if lang != "tr":539 text = re.sub(_decimal_number_re, lambda m: _expand_decimal_point(m, lang), text)540 text = re.sub(_ordinal_re[lang], lambda m: _expand_ordinal(m, lang), text)541 text = re.sub(_number_re, lambda m: _expand_number(m, lang), text)542 return text543 544 545def lowercase(text):546 return text.lower()547 548 549def collapse_whitespace(text):550 return re.sub(_whitespace_re, " ", text)551 552 553def multilingual_cleaners(text, lang):554 text = text.replace('"', "")555 if lang == "tr":556 text = text.replace("İ", "i")557 text = text.replace("Ö", "ö")558 text = text.replace("Ü", "ü")559 text = lowercase(text)560 try:561 text = expand_numbers_multilingual(text, lang)562 except:563 pass564 try:565 text = expand_abbreviations_multilingual(text, lang)566 except:567 pass568 try:569 text = expand_symbols_multilingual(text, lang=lang)570 except:571 pass572 text = collapse_whitespace(text)573 return text574 575 576def basic_cleaners(text):577 """Basic pipeline that lowercases and collapses whitespace without transliteration."""578 text = lowercase(text)579 text = collapse_whitespace(text)580 return text581 582 583def chinese_transliterate(text):584 return "".join(585 [p[0] for p in pypinyin.pinyin(text, style=pypinyin.Style.TONE3, heteronym=False, neutral_tone_with_five=True)]586 )587 588 589def japanese_cleaners(text, katsu):590 text = katsu.romaji(text)591 text = lowercase(text)592 return text593 594 595def korean_transliterate(text):596 r = Transliter(academic)597 return r.translit(text)598 599 600DEFAULT_VOCAB_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vocab.json")601 602 603class VoiceBpeTokenizer:604 def __init__(self, vocab_file=DEFAULT_VOCAB_FILE):605 self.tokenizer = None606 if vocab_file is not None:607 self.tokenizer = Tokenizer.from_file(vocab_file)608 self.char_limits = {609 "en": 10000,610 "de": 253,611 "fr": 273,612 "es": 239,613 "it": 213,614 "pt": 203,615 "pl": 224,616 "zh": 82,617 "ar": 166,618 "cs": 186,619 "ru": 182,620 "nl": 251,621 "tr": 226,622 "ja": 71,623 "hu": 224,624 "ko": 95,625 }626 627 @cached_property628 def katsu(self):629 import cutlet630 631 return cutlet.Cutlet()632 633 def check_input_length(self, txt, lang):634 lang = lang.split("-")[0] # remove the region635 limit = self.char_limits.get(lang, 250)636 # if len(txt) > limit:637 # print(638 # f"[!] Warning: The text length exceeds the character limit of {limit} for language '{lang}', this might cause truncated audio."639 # )640 641 def preprocess_text(self, txt, lang):642 if lang in {"ar", "cs", "de", "en", "es", "fr", "hu", "it", "nl", "pl", "pt", "ru", "tr", "zh", "ko"}:643 txt = multilingual_cleaners(txt, lang)644 if lang == "zh":645 txt = chinese_transliterate(txt)646 if lang == "ko":647 txt = korean_transliterate(txt)648 elif lang == "ja":649 txt = japanese_cleaners(txt, self.katsu)650 elif lang == "hi":651 # @manmay will implement this652 txt = basic_cleaners(txt)653 else:654 raise NotImplementedError(f"Language '{lang}' is not supported.")655 return txt656 657 def encode(self, txt, lang):658 lang = lang.split("-")[0] # remove the region659 self.check_input_length(txt, lang)660 txt = self.preprocess_text(txt, lang)661 lang = "zh-cn" if lang == "zh" else lang662 txt = f"[{lang}]{txt}"663 txt = txt.replace(" ", "[SPACE]")664 return self.tokenizer.encode(txt).ids665 666 def decode(self, seq, skip_special_tokens=False):667 if isinstance(seq, torch.Tensor):668 seq = seq.cpu().numpy()669 txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "")670 txt = txt.replace("[SPACE]", " ")671 txt = txt.replace("[STOP]", "")672 # txt = txt.replace("[UNK]", "")673 return txt674 675 676 #copy from https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3936677 def batch_decode(678 self,679 sequences: Union[List[int], List[List[int]], "np.ndarray", "torch.Tensor", "tf.Tensor"],680 skip_special_tokens: bool = False,681 ) -> List[str]:682 """683 Convert a list of lists of token ids into a list of strings by calling decode.684 685 Args:686 sequences (`Union[List[int], List[List[int]], np.ndarray, torch.Tensor, tf.Tensor]`):687 List of tokenized input ids. Can be obtained using the `__call__` method.688 skip_special_tokens (`bool`, *optional*, defaults to `False`):689 Whether or not to remove special tokens in the decoding.690 kwargs (additional keyword arguments, *optional*):691 Will be passed to the underlying model specific decode method.692 693 Returns:694 `List[str]`: The list of decoded sentences.695 """696 return [697 self.decode(seq)698 for seq in sequences699 ]700 701 #https://github.com/coqui-ai/TTS/blob/dev/TTS/tts/layers/xtts/trainer/dataset.py#L202702 # def pad(self): 703 704 def __len__(self):705 return self.tokenizer.get_vocab_size()706 707 def get_number_tokens(self):708 return max(self.tokenizer.get_vocab().values()) + 1709 710 711def test_expand_numbers_multilingual():712 test_cases = [713 # English714 ("In 12.5 seconds.", "In twelve point five seconds.", "en"),715 ("There were 50 soldiers.", "There were fifty soldiers.", "en"),716 ("This is a 1st test", "This is a first test", "en"),717 ("That will be $20 sir.", "That will be twenty dollars sir.", "en"),718 ("That will be 20€ sir.", "That will be twenty euro sir.", "en"),719 ("That will be 20.15€ sir.", "That will be twenty euro, fifteen cents sir.", "en"),720 ("That's 100,000.5.", "That's one hundred thousand point five.", "en"),721 # French722 ("En 12,5 secondes.", "En douze virgule cinq secondes.", "fr"),723 ("Il y avait 50 soldats.", "Il y avait cinquante soldats.", "fr"),724 ("Ceci est un 1er test", "Ceci est un premier test", "fr"),725 ("Cela vous fera $20 monsieur.", "Cela vous fera vingt dollars monsieur.", "fr"),726 ("Cela vous fera 20€ monsieur.", "Cela vous fera vingt euros monsieur.", "fr"),727 ("Cela vous fera 20,15€ monsieur.", "Cela vous fera vingt euros et quinze centimes monsieur.", "fr"),728 ("Ce sera 100.000,5.", "Ce sera cent mille virgule cinq.", "fr"),729 # German730 ("In 12,5 Sekunden.", "In zwölf Komma fünf Sekunden.", "de"),731 ("Es gab 50 Soldaten.", "Es gab fünfzig Soldaten.", "de"),732 ("Dies ist ein 1. Test", "Dies ist ein erste Test", "de"), # Issue with gender733 ("Das macht $20 Herr.", "Das macht zwanzig Dollar Herr.", "de"),734 ("Das macht 20€ Herr.", "Das macht zwanzig Euro Herr.", "de"),735 ("Das macht 20,15€ Herr.", "Das macht zwanzig Euro und fünfzehn Cent Herr.", "de"),736 # Spanish737 ("En 12,5 segundos.", "En doce punto cinco segundos.", "es"),738 ("Había 50 soldados.", "Había cincuenta soldados.", "es"),739 ("Este es un 1er test", "Este es un primero test", "es"),740 ("Eso le costará $20 señor.", "Eso le costará veinte dólares señor.", "es"),741 ("Eso le costará 20€ señor.", "Eso le costará veinte euros señor.", "es"),742 ("Eso le costará 20,15€ señor.", "Eso le costará veinte euros con quince céntimos señor.", "es"),743 # Italian744 ("In 12,5 secondi.", "In dodici virgola cinque secondi.", "it"),745 ("C'erano 50 soldati.", "C'erano cinquanta soldati.", "it"),746 ("Questo è un 1° test", "Questo è un primo test", "it"),747 ("Ti costerà $20 signore.", "Ti costerà venti dollari signore.", "it"),748 ("Ti costerà 20€ signore.", "Ti costerà venti euro signore.", "it"),749 ("Ti costerà 20,15€ signore.", "Ti costerà venti euro e quindici centesimi signore.", "it"),750 # Portuguese751 ("Em 12,5 segundos.", "Em doze vírgula cinco segundos.", "pt"),752 ("Havia 50 soldados.", "Havia cinquenta soldados.", "pt"),753 ("Este é um 1º teste", "Este é um primeiro teste", "pt"),754 ("Isso custará $20 senhor.", "Isso custará vinte dólares senhor.", "pt"),755 ("Isso custará 20€ senhor.", "Isso custará vinte euros senhor.", "pt"),756 (757 "Isso custará 20,15€ senhor.",758 "Isso custará vinte euros e quinze cêntimos senhor.",759 "pt",760 ), # "cêntimos" should be "centavos" num2words issue761 # Polish762 ("W 12,5 sekundy.", "W dwanaście przecinek pięć sekundy.", "pl"),763 ("Było 50 żołnierzy.", "Było pięćdziesiąt żołnierzy.", "pl"),764 ("To będzie kosztować 20€ panie.", "To będzie kosztować dwadzieścia euro panie.", "pl"),765 ("To będzie kosztować 20,15€ panie.", "To będzie kosztować dwadzieścia euro, piętnaście centów panie.", "pl"),766 # Arabic767 ("في الـ 12,5 ثانية.", "في الـ اثنا عشر , خمسون ثانية.", "ar"),768 ("كان هناك 50 جنديًا.", "كان هناك خمسون جنديًا.", "ar"),769 # ("ستكون النتيجة $20 يا سيد.", 'ستكون النتيجة عشرون دولار يا سيد.', 'ar'), # $ and € are mising from num2words770 # ("ستكون النتيجة 20€ يا سيد.", 'ستكون النتيجة عشرون يورو يا سيد.', 'ar'),771 # Czech772 ("Za 12,5 vteřiny.", "Za dvanáct celá pět vteřiny.", "cs"),773 ("Bylo tam 50 vojáků.", "Bylo tam padesát vojáků.", "cs"),774 ("To bude stát 20€ pane.", "To bude stát dvacet euro pane.", "cs"),775 ("To bude 20.15€ pane.", "To bude dvacet euro, patnáct centů pane.", "cs"),776 # Russian777 ("Через 12.5 секунды.", "Через двенадцать запятая пять секунды.", "ru"),778 ("Там было 50 солдат.", "Там было пятьдесят солдат.", "ru"),779 ("Это будет 20.15€ сэр.", "Это будет двадцать евро, пятнадцать центов сэр.", "ru"),780 ("Это будет стоить 20€ господин.", "Это будет стоить двадцать евро господин.", "ru"),781 # Dutch782 ("In 12,5 seconden.", "In twaalf komma vijf seconden.", "nl"),783 ("Er waren 50 soldaten.", "Er waren vijftig soldaten.", "nl"),784 ("Dat wordt dan $20 meneer.", "Dat wordt dan twintig dollar meneer.", "nl"),785 ("Dat wordt dan 20€ meneer.", "Dat wordt dan twintig euro meneer.", "nl"),786 # Chinese (Simplified)787 ("在12.5秒内", "在十二点五秒内", "zh"),788 ("有50名士兵", "有五十名士兵", "zh"),789 # ("那将是$20先生", '那将是二十美元先生', 'zh'), currency doesn't work790 # ("那将是20€先生", '那将是二十欧元先生', 'zh'),791 # Turkish792 # ("12,5 saniye içinde.", 'On iki virgül beş saniye içinde.', 'tr'), # decimal doesn't work for TR793 ("50 asker vardı.", "elli asker vardı.", "tr"),794 ("Bu 1. test", "Bu birinci test", "tr"),795 # ("Bu 100.000,5.", 'Bu yüz bin virgül beş.', 'tr'),796 # Hungarian797 ("12,5 másodperc alatt.", "tizenkettő egész öt tized másodperc alatt.", "hu"),798 ("50 katona volt.", "ötven katona volt.", "hu"),799 ("Ez az 1. teszt", "Ez az első teszt", "hu"),800 # Korean801 ("12.5 초 안에.", "십이 점 다섯 초 안에.", "ko"),802 ("50 명의 병사가 있었다.", "오십 명의 병사가 있었다.", "ko"),803 ("이것은 1 번째 테스트입니다", "이것은 첫 번째 테스트입니다", "ko"),804 ]805 for a, b, lang in test_cases:806 out = expand_numbers_multilingual(a, lang=lang)807 assert out == b, f"'{out}' vs '{b}'"808 809 810def test_abbreviations_multilingual():811 test_cases = [812 # English813 ("Hello Mr. Smith.", "Hello mister Smith.", "en"),814 ("Dr. Jones is here.", "doctor Jones is here.", "en"),815 # Spanish816 ("Hola Sr. Garcia.", "Hola señor Garcia.", "es"),817 ("La Dra. Martinez es muy buena.", "La doctora Martinez es muy buena.", "es"),818 # French819 ("Bonjour Mr. Dupond.", "Bonjour monsieur Dupond.", "fr"),820 ("Mme. Moreau est absente aujourd'hui.", "madame Moreau est absente aujourd'hui.", "fr"),821 # German822 ("Frau Dr. Müller ist sehr klug.", "Frau doktor Müller ist sehr klug.", "de"),823 # Portuguese824 ("Olá Sr. Silva.", "Olá senhor Silva.", "pt"),825 ("Dra. Costa, você está disponível?", "doutora Costa, você está disponível?", "pt"),826 # Italian827 ("Buongiorno, Sig. Rossi.", "Buongiorno, signore Rossi.", "it"),828 # ("Sig.ra Bianchi, posso aiutarti?", 'signora Bianchi, posso aiutarti?', 'it'), # Issue with matching that pattern829 # Polish830 ("Dzień dobry, P. Kowalski.", "Dzień dobry, pani Kowalski.", "pl"),831 ("M. Nowak, czy mogę zadać pytanie?", "pan Nowak, czy mogę zadać pytanie?", "pl"),832 # Czech833 ("P. Novák", "pan Novák", "cs"),834 ("Dr. Vojtěch", "doktor Vojtěch", "cs"),835 # Dutch836 ("Dhr. Jansen", "de heer Jansen", "nl"),837 ("Mevr. de Vries", "mevrouw de Vries", "nl"),838 # Russian839 ("Здравствуйте Г-н Иванов.", "Здравствуйте господин Иванов.", "ru"),840 ("Д-р Смирнов здесь, чтобы увидеть вас.", "доктор Смирнов здесь, чтобы увидеть вас.", "ru"),841 # Turkish842 ("Merhaba B. Yılmaz.", "Merhaba bay Yılmaz.", "tr"),843 ("Dr. Ayşe burada.", "doktor Ayşe burada.", "tr"),844 # Hungarian845 ("Dr. Szabó itt van.", "doktor Szabó itt van.", "hu"),846 ]847 848 for a, b, lang in test_cases:849 out = expand_abbreviations_multilingual(a, lang=lang)850 assert out == b, f"'{out}' vs '{b}'"851 852 853def test_symbols_multilingual():854 test_cases = [855 ("I have 14% battery", "I have 14 percent battery", "en"),856 ("Te veo @ la fiesta", "Te veo arroba la fiesta", "es"),857 ("J'ai 14° de fièvre", "J'ai 14 degrés de fièvre", "fr"),858 ("Die Rechnung beträgt £ 20", "Die Rechnung beträgt pfund 20", "de"),859 ("O meu email é ana&joao@gmail.com", "O meu email é ana e joao arroba gmail.com", "pt"),860 ("linguaggio di programmazione C#", "linguaggio di programmazione C cancelletto", "it"),861 ("Moja temperatura to 36.6°", "Moja temperatura to 36.6 stopnie", "pl"),862 ("Mám 14% baterie", "Mám 14 procento baterie", "cs"),863 ("Těším se na tebe @ party", "Těším se na tebe na party", "cs"),864 ("У меня 14% заряда", "У меня 14 процентов заряда", "ru"),865 ("Я буду @ дома", "Я буду собака дома", "ru"),866 ("Ik heb 14% batterij", "Ik heb 14 procent batterij", "nl"),867 ("Ik zie je @ het feest", "Ik zie je bij het feest", "nl"),868 ("لدي 14% في البطارية", "لدي 14 في المئة في البطارية", "ar"),869 ("我的电量为 14%", "我的电量为 14 百分之", "zh"),870 ("Pilim %14 dolu.", "Pilim yüzde 14 dolu.", "tr"),871 ("Az akkumulátorom töltöttsége 14%", "Az akkumulátorom töltöttsége 14 százalék", "hu"),872 ("배터리 잔량이 14%입니다.", "배터리 잔량이 14 퍼센트입니다.", "ko"),873 ]874 875 for a, b, lang in test_cases:876 out = expand_symbols_multilingual(a, lang=lang)877 assert out == b, f"'{out}' vs '{b}'"878 879 880if __name__ == "__main__":881 test_expand_numbers_multilingual()882 test_abbreviations_multilingual()883 test_symbols_multilingual()