sumanthd/IndicTrans-MultilingualTranslation

1import sys2import codecs3 4def clean_vocab(in_vocab_fname, out_vocab_fname):5    with codecs.open(in_vocab_fname, "r", encoding="utf-8") as infile, codecs.open(6        out_vocab_fname, "w", encoding="utf-8"7    ) as outfile:8        for i, line in enumerate(infile):9            fields = line.strip("\r\n ").split(" ")10            if len(fields) == 2:11                outfile.write(line)12            if len(fields) != 2:13                print("{}: {}".format(i, line.strip()))14                for c in line:15                    print("{}:{}".format(c, hex(ord(c))))16 17 18if __name__ == "__main__":19    clean_vocab(sys.argv[1], sys.argv[2])20