RASMUS/Finnish-ASR-Canary-v2
01.2k
1# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import argparse16import base6417import csv18import datetime19import difflib20import io21import json22import logging23import math24import operator25import os26import pickle27from collections import defaultdict28from os.path import expanduser29from pathlib import Path30 31import dash32import dash_bootstrap_components as dbc33import diff_match_patch34import editdistance35import jiwer36import librosa37import numpy as np38import pandas as pd39import soundfile as sf40import tqdm41from dash import dash_table, dcc, html42from dash.dependencies import Input, Output, State43from dash.exceptions import PreventUpdate44from plotly import express as px45from plotly import graph_objects as go46from plotly.subplots import make_subplots47 48# number of items in a table per page49DATA_PAGE_SIZE = 1050 51# operators for filtering items52filter_operators = {53 '>=': 'ge',54 '<=': 'le',55 '<': 'lt',56 '>': 'gt',57 '!=': 'ne',58 '=': 'eq',59 'contains ': 'contains',60}61comparison_mode = False62 63 64# parse table filter queries65def split_filter_part(filter_part):66 for op in filter_operators:67 if op in filter_part:68 name_part, value_part = filter_part.split(op, 1)69 name = name_part[name_part.find('{') + 1 : name_part.rfind('}')]70 value_part = value_part.strip()71 v0 = value_part[0]72 if v0 == value_part[-1] and v0 in ("'", '"', '`'):73 value = value_part[1:-1].replace('\\' + v0, v0)74 else:75 try:76 value = float(value_part)77 except ValueError:78 value = value_part79 return name, filter_operators[op], value80 return [None] * 381 82 83# standard command-line arguments parser84def parse_args():85 parser = argparse.ArgumentParser(description='Speech Data Explorer')86 parser.add_argument(87 'manifest',88 help='path to JSON manifest file',89 )90 parser.add_argument('--vocab', help='optional vocabulary to highlight OOV words')91 parser.add_argument('--port', default='8050', help='serving port for establishing connection')92 parser.add_argument(93 '--disable-caching-metrics', action='store_true', help='disable caching metrics for errors analysis'94 )95 parser.add_argument(96 '--estimate-audio-metrics',97 '-a',98 action='store_true',99 help='estimate frequency bandwidth and signal level of audio recordings',100 )101 parser.add_argument(102 '--audio-base-path',103 default=None,104 type=str,105 help='A base path for the relative paths in manifest. It defaults to manifest path.',106 )107 parser.add_argument('--debug', '-d', action='store_true', help='enable debug mode')108 109 parser.add_argument(110 '--names_compared',111 '-nc',112 nargs=2,113 type=str,114 help='names of the two fields that will be compared, example: pred_text_contextnet pred_text_conformer. "pred_text_" prefix IS IMPORTANT!',115 )116 parser.add_argument(117 '--show_statistics',118 '-shst',119 type=str,120 help='field name for which you want to see statistics (optional). Example: pred_text_contextnet.',121 )122 args = parser.parse_args()123 124 # assume audio_filepath is relative to the directory where the manifest is stored125 if args.audio_base_path is None:126 args.audio_base_path = os.path.dirname(args.manifest)127 128 # automaticly going in comparison mode, if there is names_compared argument129 if args.names_compared is not None:130 comparison_mode = True131 logging.info("comparison mod set to true")132 else:133 comparison_mode = False134 135 print(args, comparison_mode)136 return args, comparison_mode137 138 139# estimate frequency bandwidth of signal140def eval_bandwidth(signal, sr, threshold=-50):141 time_stride = 0.01142 hop_length = int(sr * time_stride)143 n_fft = 512144 spectrogram = np.mean(145 np.abs(librosa.stft(y=signal, n_fft=n_fft, hop_length=hop_length, window='blackmanharris')) ** 2, axis=1146 )147 power_spectrum = librosa.power_to_db(S=spectrogram, ref=np.max, top_db=100)148 freqband = 0149 for idx in range(len(power_spectrum) - 1, -1, -1):150 if power_spectrum[idx] > threshold:151 freqband = idx / n_fft * sr152 break153 return freqband154 155 156# load data from JSON manifest file157def load_data(158 data_filename,159 disable_caching=False,160 estimate_audio=False,161 vocab=None,162 audio_base_path=None,163 comparison_mode=False,164 names=None,165):166 if comparison_mode:167 if names is None:168 logging.error(f'Please, specify names of compared models')169 name_1, name_2 = names170 171 if not comparison_mode:172 if vocab is not None:173 # load external vocab174 vocabulary_ext = {}175 with open(vocab, 'r') as f:176 for line in f:177 if '\t' in line:178 # parse word from TSV file179 word = line.split('\t')[0]180 else:181 # assume each line contains just a single word182 word = line.strip()183 vocabulary_ext[word] = 1184 185 if not disable_caching:186 pickle_filename = data_filename.split('.json')[0]187 json_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(data_filename))188 timestamp = json_mtime.strftime('%Y%m%d_%H%M')189 pickle_filename += '_' + timestamp + '.pkl'190 if os.path.exists(pickle_filename):191 with open(pickle_filename, 'rb') as f:192 data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available = pickle.load(f)193 if vocab is not None:194 for item in vocabulary_data:195 item['OOV'] = item['word'] not in vocabulary_ext196 if estimate_audio:197 for item in data:198 filepath = absolute_audio_filepath(item['audio_filepath'], audio_base_path)199 signal, sr = librosa.load(path=filepath, sr=None)200 bw = eval_bandwidth(signal, sr)201 item['freq_bandwidth'] = int(bw)202 item['level_db'] = 20 * np.log10(np.max(np.abs(signal)))203 with open(pickle_filename, 'wb') as f:204 pickle.dump(205 [data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available],206 f,207 pickle.HIGHEST_PROTOCOL,208 )209 return data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available210 211 data = []212 wer_count = 0213 cer_count = 0214 wmr_count = 0215 wer = 0216 cer = 0217 wmr = 0218 mwa = 0219 num_hours = 0220 match_vocab_1 = defaultdict(lambda: 0)221 match_vocab_2 = defaultdict(lambda: 0)222 223 def append_data(224 data_filename,225 estimate_audio,226 field_name='pred_text',227 ):228 data = []229 wer_dist = 0.0230 wer_count = 0231 cer_dist = 0.0232 cer_count = 0233 wmr_count = 0234 wer = 0235 cer = 0236 wmr = 0237 mwa = 0238 num_hours = 0239 vocabulary = defaultdict(lambda: 0)240 alphabet = set()241 match_vocab = defaultdict(lambda: 0)242 243 sm = difflib.SequenceMatcher()244 metrics_available = False245 with open(data_filename, 'r', encoding='utf8') as f:246 for line in tqdm.tqdm(f):247 item = json.loads(line)248 if not isinstance(item['text'], str):249 item['text'] = ''250 num_chars = len(item['text'])251 orig = item['text'].split()252 num_words = len(orig)253 for word in orig:254 vocabulary[word] += 1255 for char in item['text']:256 alphabet.add(char)257 num_hours += item['duration']258 259 if field_name in item:260 metrics_available = True261 pred = item[field_name].split()262 measures = jiwer.compute_measures(item['text'], item[field_name])263 word_dist = measures['substitutions'] + measures['insertions'] + measures['deletions']264 char_dist = editdistance.eval(item['text'], item[field_name])265 wer_dist += word_dist266 cer_dist += char_dist267 wer_count += num_words268 cer_count += num_chars269 270 sm.set_seqs(orig, pred)271 for m in sm.get_matching_blocks():272 for word_idx in range(m[0], m[0] + m[2]):273 match_vocab[orig[word_idx]] += 1274 wmr_count += measures['hits']275 else:276 if comparison_mode:277 if field_name != 'pred_text':278 if field_name == name_1:279 logging.error(f"The .json file has no field with name: {name_1}")280 exit()281 if field_name == name_2:282 logging.error(f"The .json file has no field with name: {name_2}")283 exit()284 data.append(285 {286 'audio_filepath': item['audio_filepath'],287 'duration': round(item['duration'], 2),288 'num_words': num_words,289 'num_chars': num_chars,290 'word_rate': round(num_words / item['duration'], 2),291 'char_rate': round(num_chars / item['duration'], 2),292 'text': item['text'],293 }294 )295 if metrics_available:296 data[-1][field_name] = item[field_name]297 if num_words == 0:298 num_words = 1e-9299 if num_chars == 0:300 num_chars = 1e-9301 data[-1]['WER'] = round(word_dist / num_words * 100.0, 2)302 data[-1]['CER'] = round(char_dist / num_chars * 100.0, 2)303 data[-1]['WMR'] = round(measures['hits'] / num_words * 100.0, 2)304 data[-1]['I'] = measures['insertions']305 data[-1]['D'] = measures['deletions']306 data[-1]['D-I'] = measures['deletions'] - measures['insertions']307 if estimate_audio:308 filepath = absolute_audio_filepath(item['audio_filepath'], data_filename)309 signal, sr = librosa.load(path=filepath, sr=None)310 bw = eval_bandwidth(signal, sr)311 item['freq_bandwidth'] = int(bw)312 item['level_db'] = 20 * np.log10(np.max(np.abs(signal)))313 for k in item:314 if k not in data[-1]:315 data[-1][k] = item[k]316 317 vocabulary_data = [{'word': word, 'count': vocabulary[word]} for word in vocabulary]318 return (319 vocabulary_data,320 metrics_available,321 data,322 wer_dist,323 wer_count,324 cer_dist,325 cer_count,326 wmr_count,327 wer,328 cer,329 wmr,330 mwa,331 num_hours,332 vocabulary,333 alphabet,334 match_vocab,335 )336 337 (338 vocabulary_data,339 metrics_available,340 data,341 wer_dist,342 wer_count,343 cer_dist,344 cer_count,345 wmr_count,346 wer,347 cer,348 wmr,349 mwa,350 num_hours,351 vocabulary,352 alphabet,353 match_vocab,354 ) = append_data(data_filename, estimate_audio, field_name=fld_nm)355 if comparison_mode:356 (357 vocabulary_data_1,358 metrics_available_1,359 data_1,360 wer_dist_1,361 wer_count_1,362 cer_dist_1,363 cer_count_1,364 wmr_count_1,365 wer_1,366 cer_1,367 wmr_1,368 mwa_1,369 num_hours_1,370 vocabulary_1,371 alphabet_1,372 match_vocab_1,373 ) = append_data(data_filename, estimate_audio, field_name=name_1)374 (375 vocabulary_data_2,376 metrics_available_2,377 data_2,378 wer_dist_2,379 wer_count_2,380 cer_dist_2,381 cer_count_2,382 wmr_count_2,383 wer_2,384 cer_2,385 wmr_2,386 mwa_2,387 num_hours_2,388 vocabulary_2,389 alphabet_2,390 match_vocab_2,391 ) = append_data(data_filename, estimate_audio, field_name=name_2)392 393 if not comparison_mode:394 if vocab is not None:395 for item in vocabulary_data:396 item['OOV'] = item['word'] not in vocabulary_ext397 398 if metrics_available or comparison_mode:399 if metrics_available:400 wer = wer_dist / wer_count * 100.0401 cer = cer_dist / cer_count * 100.0402 wmr = wmr_count / wer_count * 100.0403 if comparison_mode:404 if metrics_available_1 and metrics_available_2:405 wer_1 = wer_dist_1 / wer_count_1 * 100.0406 cer_1 = cer_dist_1 / cer_count_1 * 100.0407 wmr_1 = wmr_count_1 / wer_count_1 * 100.0408 409 wer = wer_dist_2 / wer_count_2 * 100.0410 cer = cer_dist_2 / cer_count_2 * 100.0411 wmr = wmr_count_2 / wer_count_2 * 100.0412 413 acc_sum_1 = 0414 acc_sum_2 = 0415 416 for item in vocabulary_data_1:417 w = item['word']418 word_accuracy_1 = match_vocab_1[w] / vocabulary_1[w] * 100.0419 acc_sum_1 += word_accuracy_1420 item['accuracy_1'] = round(word_accuracy_1, 1)421 mwa_1 = acc_sum_1 / len(vocabulary_data_1)422 423 for item in vocabulary_data_2:424 w = item['word']425 word_accuracy_2 = match_vocab_2[w] / vocabulary_2[w] * 100.0426 acc_sum_2 += word_accuracy_2427 item['accuracy_2'] = round(word_accuracy_2, 1)428 mwa_2 = acc_sum_2 / len(vocabulary_data_2)429 430 acc_sum = 0431 for item in vocabulary_data:432 w = item['word']433 word_accuracy = match_vocab[w] / vocabulary[w] * 100.0434 acc_sum += word_accuracy435 item['accuracy'] = round(word_accuracy, 1)436 mwa = acc_sum / len(vocabulary_data)437 438 num_hours /= 3600.0439 440 if not comparison_mode:441 if not disable_caching:442 with open(pickle_filename, 'wb') as f:443 pickle.dump(444 [data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available],445 f,446 pickle.HIGHEST_PROTOCOL,447 )448 if comparison_mode:449 return (450 data,451 wer,452 cer,453 wmr,454 mwa,455 num_hours,456 vocabulary_data,457 alphabet,458 metrics_available,459 data_1,460 wer_1,461 cer_1,462 wmr_1,463 mwa_1,464 num_hours_1,465 vocabulary_data_1,466 alphabet_1,467 metrics_available_1,468 data_2,469 wer_2,470 cer_2,471 wmr_2,472 mwa_2,473 num_hours_2,474 vocabulary_data_2,475 alphabet_2,476 metrics_available_2,477 )478 479 return data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available480 481 482# plot histogram of specified field in data list483def plot_histogram(data, key, label):484 fig = px.histogram(485 data_frame=[item[key] for item in data],486 nbins=50,487 log_y=True,488 labels={'value': label},489 opacity=0.5,490 color_discrete_sequence=['green'],491 height=200,492 )493 fig.update_layout(showlegend=False, margin=dict(l=0, r=0, t=0, b=0, pad=0))494 return fig495 496 497def plot_word_accuracy(vocabulary_data):498 labels = ['Unrecognized', 'Sometimes recognized', 'Always recognized']499 counts = [0, 0, 0]500 for word in vocabulary_data:501 if word['accuracy'] == 0:502 counts[0] += 1503 elif word['accuracy'] < 100:504 counts[1] += 1505 else:506 counts[2] += 1507 colors = ['red', 'orange', 'green']508 509 fig = go.Figure(510 data=[511 go.Bar(512 x=labels,513 y=counts,514 marker_color=colors,515 text=['{:.2%}'.format(count / sum(counts)) for count in counts],516 textposition='auto',517 )518 ]519 )520 fig.update_layout(521 showlegend=False, margin=dict(l=0, r=0, t=0, b=0, pad=0), height=200, yaxis={'title_text': '#words'}522 )523 524 return fig525 526 527def absolute_audio_filepath(audio_filepath, audio_base_path):528 """Return absolute path to an audio file.529 530 Check if a file existst at audio_filepath.531 If not, assume that the path is relative to audio_base_path.532 """533 audio_filepath = Path(audio_filepath)534 535 if not audio_filepath.is_file() and not audio_filepath.is_absolute():536 audio_filepath = audio_base_path / audio_filepath537 if audio_filepath.is_file():538 filename = str(audio_filepath)539 else:540 filename = expanduser(audio_filepath)541 else:542 filename = expanduser(audio_filepath)543 544 return filename545 546 547# parse the CLI arguments548args, comparison_mode = parse_args()549if args.show_statistics is not None:550 fld_nm = args.show_statistics551else:552 fld_nm = 'pred_text'553# parse names of compared models, if any554if comparison_mode:555 name_1, name_2 = args.names_compared556 print(name_1, name_2)557 558 559print('Loading data...')560if not comparison_mode:561 data, wer, cer, wmr, mwa, num_hours, vocabulary, alphabet, metrics_available = load_data(562 args.manifest,563 args.disable_caching_metrics,564 args.estimate_audio_metrics,565 args.vocab,566 args.audio_base_path,567 comparison_mode,568 args.names_compared,569 )570else:571 (572 data,573 wer,574 cer,575 wmr,576 mwa,577 num_hours,578 vocabulary,579 alphabet,580 metrics_available,581 data_1,582 wer_1,583 cer_1,584 wmr_1,585 mwa_1,586 num_hours_1,587 vocabulary_1,588 alphabet_1,589 metrics_available_1,590 data_2,591 wer_2,592 cer_2,593 wmr_2,594 mwa_2,595 num_hours_2,596 vocabulary_2,597 alphabet_2,598 metrics_available_2,599 ) = load_data(600 args.manifest,601 args.disable_caching_metrics,602 args.estimate_audio_metrics,603 args.vocab,604 args.audio_base_path,605 comparison_mode,606 args.names_compared,607 )608 609print('Starting server...')610app = dash.Dash(611 __name__,612 suppress_callback_exceptions=True,613 external_stylesheets=[dbc.themes.BOOTSTRAP],614 title=os.path.basename(args.manifest),615)616 617figures_labels = {618 'duration': ['Duration', 'Duration, sec'],619 'num_words': ['Number of Words', '#words'],620 'num_chars': ['Number of Characters', '#chars'],621 'word_rate': ['Word Rate', '#words/sec'],622 'char_rate': ['Character Rate', '#chars/sec'],623 'WER': ['Word Error Rate', 'WER, %'],624 'CER': ['Character Error Rate', 'CER, %'],625 'WMR': ['Word Match Rate', 'WMR, %'],626 'I': ['# Insertions (I)', '#words'],627 'D': ['# Deletions (D)', '#words'],628 'D-I': ['# Deletions - # Insertions (D-I)', '#words'],629 'freq_bandwidth': ['Frequency Bandwidth', 'Bandwidth, Hz'],630 'level_db': ['Peak Level', 'Level, dB'],631}632figures_hist = {}633for k in data[0]:634 val = data[0][k]635 if isinstance(val, (int, float)) and not isinstance(val, bool):636 if k in figures_labels:637 ylabel = figures_labels[k][0]638 xlabel = figures_labels[k][1]639 else:640 title = k.replace('_', ' ')641 title = title[0].upper() + title[1:].lower()642 ylabel = title643 xlabel = title644 figures_hist[k] = [ylabel + ' (per utterance)', plot_histogram(data, k, xlabel)]645 646if metrics_available:647 figure_word_acc = plot_word_accuracy(vocabulary)648 649stats_layout = [650 dbc.Row(dbc.Col(html.H5(children='Global Statistics'), class_name='text-secondary'), class_name='mt-3'),651 dbc.Row(652 [653 dbc.Col(html.Div('Number of hours', className='text-secondary'), width=3, class_name='border-end'),654 dbc.Col(html.Div('Number of utterances', className='text-secondary'), width=3, class_name='border-end'),655 dbc.Col(html.Div('Vocabulary size', className='text-secondary'), width=3, class_name='border-end'),656 dbc.Col(html.Div('Alphabet size', className='text-secondary'), width=3),657 ],658 class_name='bg-light mt-2 rounded-top border-top border-start border-end',659 ),660 dbc.Row(661 [662 dbc.Col(663 html.H5(664 '{:.2f} hours'.format(num_hours),665 className='text-center p-1',666 style={'color': 'green', 'opacity': 0.7},667 ),668 width=3,669 class_name='border-end',670 ),671 dbc.Col(672 html.H5(len(data), className='text-center p-1', style={'color': 'green', 'opacity': 0.7}),673 width=3,674 class_name='border-end',675 ),676 dbc.Col(677 html.H5(678 '{} words'.format(len(vocabulary)),679 className='text-center p-1',680 style={'color': 'green', 'opacity': 0.7},681 ),682 width=3,683 class_name='border-end',684 ),685 dbc.Col(686 html.H5(687 '{} chars'.format(len(alphabet)),688 className='text-center p-1',689 style={'color': 'green', 'opacity': 0.7},690 ),691 width=3,692 ),693 ],694 class_name='bg-light rounded-bottom border-bottom border-start border-end',695 ),696]697if metrics_available:698 stats_layout += [699 dbc.Row(700 [701 dbc.Col(702 html.Div('Word Error Rate (WER), %', className='text-secondary'), width=3, class_name='border-end'703 ),704 dbc.Col(705 html.Div('Character Error Rate (CER), %', className='text-secondary'),706 width=3,707 class_name='border-end',708 ),709 dbc.Col(710 html.Div('Word Match Rate (WMR), %', className='text-secondary'),711 width=3,712 class_name='border-end',713 ),714 dbc.Col(html.Div('Mean Word Accuracy, %', className='text-secondary'), width=3),715 ],716 class_name='bg-light mt-2 rounded-top border-top border-start border-end',717 ),718 dbc.Row(719 [720 dbc.Col(721 html.H5(722 '{:.2f}'.format(wer),723 className='text-center p-1',724 style={'color': 'green', 'opacity': 0.7},725 ),726 width=3,727 class_name='border-end',728 ),729 dbc.Col(730 html.H5(731 '{:.2f}'.format(cer), className='text-center p-1', style={'color': 'green', 'opacity': 0.7}732 ),733 width=3,734 class_name='border-end',735 ),736 dbc.Col(737 html.H5(738 '{:.2f}'.format(wmr),739 className='text-center p-1',740 style={'color': 'green', 'opacity': 0.7},741 ),742 width=3,743 class_name='border-end',744 ),745 dbc.Col(746 html.H5(747 '{:.2f}'.format(mwa),748 className='text-center p-1',749 style={'color': 'green', 'opacity': 0.7},750 ),751 width=3,752 ),753 ],754 class_name='bg-light rounded-bottom border-bottom border-start border-end',755 ),756 ]757stats_layout += [758 dbc.Row(dbc.Col(html.H5(children='Alphabet'), class_name='text-secondary'), class_name='mt-3'),759 dbc.Row(760 dbc.Col(761 html.Div('{}'.format(sorted(alphabet))),762 ),763 class_name='mt-2 bg-light font-monospace rounded border',764 ),765]766for k in figures_hist:767 stats_layout += [768 dbc.Row(dbc.Col(html.H5(figures_hist[k][0]), class_name='text-secondary'), class_name='mt-3'),769 dbc.Row(770 dbc.Col(771 dcc.Graph(id='duration-graph', figure=figures_hist[k][1]),772 ),773 ),774 ]775 776if metrics_available:777 stats_layout += [778 dbc.Row(dbc.Col(html.H5('Word accuracy distribution'), class_name='text-secondary'), class_name='mt-3'),779 dbc.Row(780 dbc.Col(781 dcc.Graph(id='word-acc-graph', figure=figure_word_acc),782 ),783 ),784 ]785 786wordstable_columns = [{'name': 'Word', 'id': 'word'}, {'name': 'Count', 'id': 'count'}]787if 'OOV' in vocabulary[0]:788 wordstable_columns.append({'name': 'OOV', 'id': 'OOV'})789if metrics_available:790 wordstable_columns.append({'name': 'Accuracy, %', 'id': 'accuracy'})791 792 793stats_layout += [794 dbc.Row(dbc.Col(html.H5('Vocabulary'), class_name='text-secondary'), class_name='mt-3'),795 dbc.Row(796 dbc.Col(797 dash_table.DataTable(798 id='wordstable',799 columns=wordstable_columns,800 filter_action='custom',801 filter_query='',802 sort_action='custom',803 sort_mode='single',804 page_action='custom',805 page_current=0,806 page_size=DATA_PAGE_SIZE,807 cell_selectable=False,808 page_count=math.ceil(len(vocabulary) / DATA_PAGE_SIZE),809 sort_by=[{'column_id': 'word', 'direction': 'asc'}],810 style_cell={'maxWidth': 0, 'textAlign': 'left'},811 style_header={'color': 'text-primary'},812 css=[813 {'selector': '.dash-filter--case', 'rule': 'display: none'},814 ],815 ),816 ),817 class_name='m-2',818 ),819 dbc.Row(820 dbc.Col(821 [822 html.Button('Download Vocabulary', id='btn_csv'),823 dcc.Download(id='download-vocab-csv'),824 ]825 ),826 ),827]828 829 830@app.callback(831 Output('download-vocab-csv', 'data'),832 [Input('btn_csv', 'n_clicks'), State('wordstable', 'sort_by'), State('wordstable', 'filter_query')],833 prevent_initial_call=True,834)835def download_vocabulary(n_clicks, sort_by, filter_query):836 vocabulary_view = vocabulary837 filtering_expressions = filter_query.split(' && ')838 for filter_part in filtering_expressions:839 col_name, op, filter_value = split_filter_part(filter_part)840 841 if op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge'):842 vocabulary_view = [x for x in vocabulary_view if getattr(operator, op)(x[col_name], filter_value)]843 elif op == 'contains':844 vocabulary_view = [x for x in vocabulary_view if filter_value in str(x[col_name])]845 846 if len(sort_by):847 col = sort_by[0]['column_id']848 descending = sort_by[0]['direction'] == 'desc'849 vocabulary_view = sorted(vocabulary_view, key=lambda x: x[col], reverse=descending)850 851 with open('sde_vocab.csv', encoding='utf-8', mode='w', newline='') as fo:852 writer = csv.writer(fo)853 writer.writerow(vocabulary_view[0].keys())854 for item in vocabulary_view:855 writer.writerow([str(item[k]) for k in item])856 return dcc.send_file("sde_vocab.csv")857 858 859@app.callback(860 [Output('wordstable', 'data'), Output('wordstable', 'page_count')],861 [Input('wordstable', 'page_current'), Input('wordstable', 'sort_by'), Input('wordstable', 'filter_query')],862)863def update_wordstable(page_current, sort_by, filter_query):864 vocabulary_view = vocabulary865 filtering_expressions = filter_query.split(' && ')866 for filter_part in filtering_expressions:867 col_name, op, filter_value = split_filter_part(filter_part)868 869 if op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge'):870 vocabulary_view = [x for x in vocabulary_view if getattr(operator, op)(x[col_name], filter_value)]871 elif op == 'contains':872 vocabulary_view = [x for x in vocabulary_view if filter_value in str(x[col_name])]873 874 if len(sort_by):875 col = sort_by[0]['column_id']876 descending = sort_by[0]['direction'] == 'desc'877 vocabulary_view = sorted(vocabulary_view, key=lambda x: x[col], reverse=descending)878 if page_current * DATA_PAGE_SIZE >= len(vocabulary_view):879 page_current = len(vocabulary_view) // DATA_PAGE_SIZE880 return [881 vocabulary_view[page_current * DATA_PAGE_SIZE : (page_current + 1) * DATA_PAGE_SIZE],882 math.ceil(len(vocabulary_view) / DATA_PAGE_SIZE),883 ]884 885 886samples_layout = [887 dbc.Row(dbc.Col(html.H5('Data'), class_name='text-secondary'), class_name='mt-3'),888 html.Hr(),889 dbc.Row(890 dbc.Col(891 dash_table.DataTable(892 id='datatable',893 columns=[{'name': k.replace('_', ' '), 'id': k, 'hideable': True} for k in data[0]],894 filter_action='custom',895 filter_query='',896 sort_action='custom',897 sort_mode='single',898 sort_by=[],899 row_selectable='single',900 selected_rows=[0],901 page_action='custom',902 page_current=0,903 page_size=DATA_PAGE_SIZE,904 page_count=math.ceil(len(data) / DATA_PAGE_SIZE),905 style_cell={'overflow': 'hidden', 'textOverflow': 'ellipsis', 'maxWidth': 0, 'textAlign': 'center'},906 style_header={907 'color': 'text-primary',908 'text_align': 'center',909 'height': 'auto',910 'whiteSpace': 'normal',911 },912 css=[913 {'selector': '.dash-spreadsheet-menu', 'rule': 'position:absolute; bottom: 8px'},914 {'selector': '.dash-filter--case', 'rule': 'display: none'},915 {'selector': '.column-header--hide', 'rule': 'display: none'},916 ],917 ),918 )919 ),920] + [921 dbc.Row(922 [923 dbc.Col(924 html.Div(children=k.replace('_', ' ')),925 width=2,926 class_name='mt-1 bg-light font-monospace text-break small rounded border',927 ),928 dbc.Col(html.Div(id='_' + k), class_name='mt-1 bg-light font-monospace text-break small rounded border'),929 ]930 )931 for k in data[0]932]933 934if metrics_available:935 samples_layout += [936 dbc.Row(937 [938 dbc.Col(939 html.Div(children='text diff'),940 width=2,941 class_name='mt-1 bg-light font-monospace text-break small rounded border',942 ),943 dbc.Col(944 html.Iframe(945 id='_diff',946 sandbox='',947 srcDoc='',948 style={'border': 'none', 'width': '100%', 'height': '100%'},949 className='bg-light font-monospace text-break small',950 ),951 class_name='mt-1 bg-light font-monospace text-break small rounded border',952 ),953 ]954 )955 ]956samples_layout += [957 dbc.Row(958 dbc.Col(959 html.Audio(id='player', controls=True),960 ),961 class_name='mt-3 ',962 ),963 dbc.Row(dbc.Col(dcc.Graph(id='signal-graph')), class_name='mt-3'),964]965 966 967# updating vocabulary to show968 969 970wordstable_columns_tool = [{'name': 'Word', 'id': 'word'}, {'name': 'Count', 'id': 'count'}]971wordstable_columns_tool.append({'name': 'Accuracy_1, %', 'id': 'accuracy_1'})972wordstable_columns_tool.append({'name': 'Accuracy_2, %', 'id': 'accuracy_2'})973 974 975if comparison_mode:976 model_name_1, model_name_2 = name_1, name_2977 978 for i in range(len(vocabulary_1)):979 vocabulary_1[i].update(vocabulary_2[i])980 981 def _wer_(grnd, pred):982 grnd_words = grnd.split()983 pred_words = pred.split()984 edit_distance = editdistance.eval(grnd_words, pred_words)985 wer = edit_distance / len(grnd_words)986 return wer987 988 def metric(a, b, met=None):989 cer = editdistance.distance(a, b) / len(a)990 wer = _wer_(a, b)991 return round(float(wer) * 100, 2), round(float(cer) * 100, 2)992 993 def write_metrics(data, Ox, Oy):994 da = pd.DataFrame.from_records(data)995 gt = da['text']996 tt_1 = da[Ox]997 tt_2 = da[Oy]998 999 wer_tt1_c, cer_tt1_c = [], []1000 wer_tt2_c, cer_tt2_c = [], []1001 1002 for j in range(len(gt)):1003 wer_tt1, cer_tt1 = metric(gt[j], tt_1[j]) # first model1004 wer_tt2, cer_tt2 = metric(gt[j], tt_2[j]) # second model1005 wer_tt1_c.append(wer_tt1)1006 cer_tt1_c.append(cer_tt1)1007 wer_tt2_c.append(wer_tt2)1008 cer_tt2_c.append(cer_tt2)1009 1010 da['wer_' + Ox] = pd.Series(wer_tt1_c, index=da.index)1011 da['wer_' + Oy] = pd.Series(wer_tt2_c, index=da.index)1012 da['cer_' + Ox] = pd.Series(cer_tt1_c, index=da.index)1013 da['cer_' + Oy] = pd.Series(cer_tt2_c, index=da.index)1014 return da.to_dict('records')1015 1016 data_with_metrics = write_metrics(data, model_name_1, model_name_2)1017 if args.show_statistics is not None:1018 textdiffstyle = {'border': 'none', 'width': '100%', 'height': '100%'}1019 else:1020 textdiffstyle = {'border': 'none', 'width': '1%', 'height': '1%', 'display': 'none'}1021 1022 def prepare_data(df, name1=model_name_1, name2=model_name_2):1023 res = pd.DataFrame()1024 tmp = df['word']1025 res.insert(0, 'word', tmp)1026 res.insert(1, 'count', [float(i) for i in df['count']])1027 res.insert(2, 'accuracy_model_' + name1, df['accuracy_1'])1028 res.insert(3, 'accuracy_model_' + name2, df['accuracy_2'])1029 res.insert(4, 'accuracy_diff ' + '(' + name1 + ' - ' + name2 + ')', df['accuracy_1'] - df['accuracy_2'])1030 res.insert(2, 'count^(-1)', 1 / df['count'])1031 return res1032 1033 for_col_names = pd.DataFrame()1034 for_col_names.insert(0, 'word', ['a'])1035 for_col_names.insert(1, 'count', [0])1036 for_col_names.insert(2, 'accuracy_model_' + model_name_1, [0])1037 for_col_names.insert(3, 'accuracy_model_' + model_name_2, [0])1038 for_col_names.insert(4, 'accuracy_diff ' + '(' + model_name_1 + ' - ' + model_name_2 + ')', [0])1039 for_col_names.insert(5, 'count^(-1)', [0])1040 1041 @app.callback(1042 Output('voc_graph', 'figure'),1043 [1044 Input('xaxis-column', 'value'),1045 Input('yaxis-column', 'value'),1046 Input('color-column', 'value'),1047 Input('size-column', 'value'),1048 Input("datatable-advanced-filtering", "derived_virtual_data"),1049 Input("dot_spacing", 'value'),1050 Input("radius", 'value'),1051 ],1052 prevent_initial_call=False,1053 )1054 def draw_vocab(Ox, Oy, color, size, data, dot_spacing='no', rad=0.01):1055 import math1056 import random1057 1058 import pandas as pd1059 1060 df = pd.DataFrame.from_records(data)1061 1062 res = prepare_data(df)1063 res_spacing = res.copy(deep=True)1064 1065 if dot_spacing == 'yes':1066 rad = float(rad)1067 if Ox[0] == 'a' or 'c':1068 tmp = []1069 for i in range(len(res[Ox])):1070 tmp.append(1071 res[Ox][i]1072 + rad1073 * random.randrange(1, 10)1074 * math.cos(random.randrange(1, len(res[Ox])) * 2 * math.pi / len(res[Ox]))1075 )1076 res_spacing[Ox] = tmp1077 if Ox[0] == 'a' or 'c':1078 tmp = []1079 for i in range(len(res[Oy])):1080 tmp.append(1081 res[Oy][i]1082 + rad1083 * random.randrange(1, 10)1084 * math.sin(random.randrange(1, len(res[Oy])) * 2 * math.pi / len(res[Oy]))1085 )1086 res_spacing[Oy] = tmp1087 1088 res = res_spacing1089 1090 fig = px.scatter(1091 res,1092 x=Ox,1093 y=Oy,1094 color=color,1095 size=size,1096 hover_data={'word': True, Ox: True, Oy: True, 'count': True},1097 width=1300,1098 height=1000,1099 )1100 if (Ox == 'accuracy_model_' + model_name_1 and Oy == 'accuracy_model_' + model_name_2) or (1101 Oy == 'accuracy_model_' + model_name_1 and Ox == 'accuracy_model_' + model_name_21102 ):1103 fig.add_shape(1104 type="line",1105 x0=0,1106 y0=0,1107 x1=100,1108 y1=100,1109 line=dict(1110 color="MediumPurple",1111 width=1,1112 dash="dot",1113 ),1114 )1115 1116 return fig1117 1118 @app.callback(1119 Output('filter-query-input', 'style'),1120 Output('filter-query-output', 'style'),1121 Input('filter-query-read-write', 'value'),1122 )1123 def query_input_output(val):1124 input_style = {'width': '100%'}1125 output_style = {}1126 input_style.update(display='inline-block')1127 output_style.update(display='none')1128 return input_style, output_style1129 1130 @app.callback(Output('datatable-advanced-filtering', 'filter_query'), Input('filter-query-input', 'value'))1131 def write_query(query):1132 if query is None:1133 return ''1134 return query1135 1136 @app.callback(Output('filter-query-output', 'children'), Input('datatable-advanced-filtering', 'filter_query'))1137 def read_query(query):1138 if query is None:1139 return "No filter query"1140 return dcc.Markdown('`filter_query = "{}"`'.format(query))1141 1142 ############1143 @app.callback(1144 Output('filter-query-input-2', 'style'),1145 Output('filter-query-output-2', 'style'),1146 Input('filter-query-read-write', 'value'),1147 )1148 def query_input_output(val):1149 input_style = {'width': '100%'}1150 output_style = {}1151 input_style.update(display='inline-block')1152 output_style.update(display='none')1153 return input_style, output_style1154 1155 @app.callback(Output('datatable-advanced-filtering-2', 'filter_query'), Input('filter-query-input-2', 'value'))1156 def write_query(query):1157 if query is None:1158 return ''1159 return query1160 1161 @app.callback(Output('filter-query-output-2', 'children'), Input('datatable-advanced-filtering-2', 'filter_query'))1162 def read_query(query):1163 if query is None:1164 return "No filter query"1165 return dcc.Markdown('`filter_query = "{}"`'.format(query))1166 1167 ############1168 1169 def display_query(query):1170 if query is None:1171 return ''1172 return html.Details(1173 [1174 html.Summary('Derived filter query structure'),1175 html.Div(1176 dcc.Markdown(1177 '''```json1178 {}1179 ```'''.format(1180 json.dumps(query, indent=4)1181 )1182 )1183 ),1184 ]1185 )1186 1187 comparison_layout = [1188 html.Div(1189 [1190 dcc.Markdown("model 1:" + ' ' + model_name_1[10:]),1191 dcc.Markdown("model 2:" + ' ' + model_name_2[10:]),1192 dcc.Dropdown(1193 ['word level', 'utterance level'],1194 'word level',1195 placeholder="choose comparison lvl",1196 id='lvl_choose',1197 ),1198 ]1199 ),1200 html.Hr(),