FrankAst/Spice
0
1# -*- coding: utf-8 -*-2"""tp3__1_-1.ipynb3 4Automatically generated by Colaboratory.5 6Original file is located at7 https://colab.research.google.com/drive/1_Sjx5G1BW689ggZJAJ4P7kCZndOobNCp8"""9 10# Install Gradio11#!pip install gradio -q12 13# Install timidy14#!sudo apt-get install -q -y timidity libsndfile115 16# All the imports to deal with sound data17#!pip install pydub numba==0.48 librosa music2118 19# Import Libraries20 21import gradio as gr22import time23 24import tensorflow as tf25import tensorflow_hub as hub26 27import numpy as np28import matplotlib.pyplot as plt29import librosa30from librosa import display as librosadisplay31 32import logging33import math34import statistics35import sys36 37from IPython.display import Audio, Javascript38from scipy.io import wavfile39 40from base64 import b64decode41 42import music2143from pydub import AudioSegment44 45logger = logging.getLogger()46logger.setLevel(logging.ERROR)47 48#print("tensorflow: %s" % tf.__version__)49#print("librosa: %s" % librosa.__version__)50 51# The audio input file52# Now the hardest part: Record your singing! :)53 54# We provide four methods to obtain an audio file:55 56# 1. Record audio directly in Gradio57# 2. Use a file saved on Google Drive58 59# Use a file saved on Google Drive60#INPUT_SOURCE = 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav'61 62#!wget --no-check-certificate 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav' -O c-scale.wav63 64#uploaded_file_name = 'c-scale.wav'65 66#uploaded_file_name67 68# Function that converts the user-created audio to the format that the model 69# expects: bitrate 16kHz and only one channel (mono).70 71EXPECTED_SAMPLE_RATE = 1600072 73# Funciones #74def convert_audio_for_model(user_file, output_file='converted_audio_file.wav'):75 audio = AudioSegment.from_file(user_file)76 audio = audio.set_frame_rate(EXPECTED_SAMPLE_RATE).set_channels(1)77 audio.export(output_file, format="wav")78 return output_file79 80MAX_ABS_INT16 = 32768.081 82def plot_stft(x, sample_rate, show_black_and_white=False):83 x_stft = np.abs(librosa.stft(x, n_fft=2048))84 fig, ax = plt.subplots()85 fig.set_size_inches(20, 10)86 x_stft_db = librosa.amplitude_to_db(x_stft, ref=np.max)87 88 if(show_black_and_white):89 librosadisplay.specshow(data=x_stft_db, 90 y_axis='log', 91 sr=sample_rate, 92 cmap='gray_r')93 else:94 librosadisplay.specshow(data=x_stft_db, 95 y_axis='log', 96 sr=sample_rate)97 98 plt.colorbar(format='%+2.0f dB')99 100 return fig101 102# Loading audio samples from the wav file:103#sample_rate, audio_samples = wavfile.read(converted_audio_file, 'rb')104 105#fig = plot_stft(audio_samples / MAX_ABS_INT16 , sample_rate=EXPECTED_SAMPLE_RATE)106 107# Executing the Model108# Loading the SPICE model is easy:109model = hub.load("https://tfhub.dev/google/spice/2")110 111def plot_pitch_conf(pitch_outputs,confidence_outputs):112 fig, ax = plt.subplots()113 fig.set_size_inches(20, 10)114 plt.plot(pitch_outputs, label='pitch')115 plt.plot(confidence_outputs, label='confidence')116 plt.legend(loc="lower right")117 return fig118 119def plot_pitch_conf_notes(confident_pitch_outputs_x,confident_pitch_outputs_y):120 fig, ax = plt.subplots()121 fig.set_size_inches(20, 10)122 ax.set_ylim([0, 1])123 plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, )124 plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, c="r")125 return fig126 127def output2hz(pitch_output):128 # Constants taken from https://tfhub.dev/google/spice/2129 PT_OFFSET = 25.58130 PT_SLOPE = 63.07131 FMIN = 10.0;132 BINS_PER_OCTAVE = 12.0;133 cqt_bin = pitch_output * PT_SLOPE + PT_OFFSET;134 return FMIN * 2.0 ** (1.0 * cqt_bin / BINS_PER_OCTAVE)135 136def espectro_notas(audio_samples,EXPECTED_SAMPLE_RATE,confident_pitch_outputs_x,confident_pitch_values_hz):137 fig, ax = plt.subplots()138 plot_stft(audio_samples / MAX_ABS_INT16 , 139 sample_rate=EXPECTED_SAMPLE_RATE, show_black_and_white=True)140 # Note: conveniently, since the plot is in log scale, the pitch outputs 141 # also get converted to the log scale automatically by matplotlib.142 plt.scatter(confident_pitch_outputs_x, confident_pitch_values_hz, c="r")143 return fig144 145def hz2offset(freq):146 # This measures the quantization error for a single note.147 if freq == 0: # Rests always have zero error.148 return None149 # Quantized note.150 h = round(12 * math.log2(freq / C0))151 return 12 * math.log2(freq / C0) - h152 153def quantize_predictions(group, ideal_offset):154 # Group values are either 0, or a pitch in Hz.155 non_zero_values = [v for v in group if v != 0]156 zero_values_count = len(group) - len(non_zero_values)157 158 # Create a rest if 80% is silent, otherwise create a note.159 if zero_values_count > 0.8 * len(group):160 # Interpret as a rest. Count each dropped note as an error, weighted a bit161 # worse than a badly sung note (which would 'cost' 0.5).162 return 0.51 * len(non_zero_values), "Rest"163 else:164 # Interpret as note, estimating as mean of non-rest predictions.165 h = round(166 statistics.mean([167 12 * math.log2(freq / C0) - ideal_offset for freq in non_zero_values168 ]))169 octave = h // 12170 n = h % 12171 note = note_names[n] + str(octave)172 # Quantization error is the total difference from the quantized note.173 error = sum([174 abs(12 * math.log2(freq / C0) - ideal_offset - h)175 for freq in non_zero_values176 ])177 return error, note178 179def get_quantization_and_error(pitch_outputs_and_rests, predictions_per_eighth,180 prediction_start_offset, ideal_offset):181 # Apply the start offset - we can just add the offset as rests.182 pitch_outputs_and_rests = [0] * prediction_start_offset + \183 pitch_outputs_and_rests184 # Collect the predictions for each note (or rest).185 groups = [186 pitch_outputs_and_rests[i:i + predictions_per_eighth]187 for i in range(0, len(pitch_outputs_and_rests), predictions_per_eighth)188 ]189 190 quantization_error = 0191 192 notes_and_rests = []193 for group in groups:194 error, note_or_rest = quantize_predictions(group, ideal_offset)195 quantization_error += error196 notes_and_rests.append(note_or_rest)197 198 return quantization_error, notes_and_rests199 200def main(audio):201 202 # Preparing the audio data203 # Now we have the audio, let's convert it to the expected format and then 204 # listen to it!205 # The SPICE model needs as input an audio file at a sampling rate of 16kHz and206 # with only one channel (mono). 207 # To help you with this part, we created a function(`convert_audio_for_model`) 208 #to convert any wav file you have to the model's expected format:209 210 211 # Converting to the expected format for the model212 # in all the input 4 input method before, the uploaded file name is at213 # the variable uploaded_file_name214 converted_audio_file = convert_audio_for_model(audio)215 216 # Loading audio samples from the wav file:217 sample_rate, audio_samples = wavfile.read(converted_audio_file, 'rb')218 219 audio_samples = audio_samples / float(MAX_ABS_INT16)220 221 222 # We now feed the audio to the SPICE tf.hub model to obtain pitch and uncertainty outputs as tensors.223 model_output = model.signatures["serving_default"](tf.constant(audio_samples, tf.float32))224 225 pitch_outputs = model_output["pitch"]226 uncertainty_outputs = model_output["uncertainty"]227 228 # 'Uncertainty' basically means the inverse of confidence.229 confidence_outputs = 1.0 - uncertainty_outputs230 231 232 confidence_outputs = list(confidence_outputs)233 pitch_outputs = [ float(x) for x in pitch_outputs]234 235 indices = range(len (pitch_outputs))236 confident_pitch_outputs = [ (i,p) 237 for i, p, c in zip(indices, pitch_outputs, confidence_outputs) if c >= 0.9 ]238 confident_pitch_outputs_x, confident_pitch_outputs_y = zip(*confident_pitch_outputs)239 240 confident_pitch_values_hz = [ output2hz(p) for p in confident_pitch_outputs_y ]241 242 243 #Plot waves244 fig1 = plt.figure()245 plt.plot(audio_samples)246 247 #Plot 248 fig2 = plot_stft(audio_samples / MAX_ABS_INT16 , sample_rate=EXPECTED_SAMPLE_RATE)249 250 #Plot Pitch & Confidence251 fig3 = plot_pitch_conf(pitch_outputs,confidence_outputs)252 253 254 #Plot Pitch & Confidence Notes255 fig4 = plot_pitch_conf_notes(confident_pitch_outputs_x,confident_pitch_outputs_y)256 257 #Plot Espectro + Notes258 fig5 = espectro_notas(audio_samples,EXPECTED_SAMPLE_RATE,confident_pitch_outputs_x,confident_pitch_values_hz)259 260 261 # ############################################################################262 # Converting to musical notes ################################################263 264 # Now that we have the pitch values, let's convert them to notes!265 # This is part is challenging by itself. We have to take into account two 266 # things:267 # 1. the rests (when there's no singing) 268 # 2. the size of each note (offsets) 269 270 # ----------------------------------------------------------------------------271 ### 1: Adding zeros to the output to indicate when there's no singing272 273 pitch_outputs_and_rests = [274 output2hz(p) if c >= 0.9 else 0275 for i, p, c in zip(indices, pitch_outputs, confidence_outputs)276 ]277 278 # ----------------------------------------------------------------------------279 ### 2: Adding note offsets280 # When a person sings freely, the melody may have an offset to the absolute 281 # pitch values that notes can represent.282 # Hence, to convert predictions to notes, one needs to correct for this 283 # possible offset.284 # This is what the following code computes.285 286 A4 = 440287 C0 = A4 * pow(2, -4.75)288 note_names = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]289 290 def hz2offset(freq):291 # This measures the quantization error for a single note.292 if freq == 0: # Rests always have zero error.293 return None294 # Quantized note.295 h = round(12 * math.log2(freq / C0))296 return 12 * math.log2(freq / C0) - h297 298 299 # The ideal offset is the mean quantization error for all the notes300 # (excluding rests):301 offsets = [hz2offset(p) for p in pitch_outputs_and_rests if p != 0]302 #print("offsets: ", offsets)303 off = offsets304 305 ideal_offset = statistics.mean(offsets)306 #print("ideal offset: ", ideal_offset)307 ideal_off = ideal_offset308 309 # We can now use some heuristics to try and estimate the most likely sequence 310 # of notes that were sung.311 # The ideal offset computed above is one ingredient - but we also need to know 312 # the speed (how many predictions make, say, an eighth?), and the time offset 313 # to start quantizing. To keep it simple, we'll just try different speeds and 314 # time offsets and measure the quantization error, using in the end the values 315 # that minimize this error.316 317 def quantize_predictions(group, ideal_offset):318 # Group values are either 0, or a pitch in Hz.319 non_zero_values = [v for v in group if v != 0]320 zero_values_count = len(group) - len(non_zero_values)321 322 # Create a rest if 80% is silent, otherwise create a note.323 if zero_values_count > 0.8 * len(group):324 # Interpret as a rest. Count each dropped note as an error, weighted a bit325 # worse than a badly sung note (which would 'cost' 0.5).326 return 0.51 * len(non_zero_values), "Rest"327 else:328 # Interpret as note, estimating as mean of non-rest predictions.329 h = round(330 statistics.mean([331 12 * math.log2(freq / C0) - ideal_offset for freq in non_zero_values332 ]))333 octave = h // 12334 n = h % 12335 note = note_names[n] + str(octave)336 # Quantization error is the total difference from the quantized note.337 error = sum([338 abs(12 * math.log2(freq / C0) - ideal_offset - h)339 for freq in non_zero_values340 ])341 return error, note342 343 344 def get_quantization_and_error(pitch_outputs_and_rests, predictions_per_eighth,345 prediction_start_offset, ideal_offset):346 # Apply the start offset - we can just add the offset as rests.347 pitch_outputs_and_rests = [0] * prediction_start_offset + \348 pitch_outputs_and_rests349 # Collect the predictions for each note (or rest).350 groups = [351 pitch_outputs_and_rests[i:i + predictions_per_eighth]352 for i in range(0, len(pitch_outputs_and_rests), predictions_per_eighth)353 ]354 355 quantization_error = 0356 357 notes_and_rests = []358 for group in groups:359 error, note_or_rest = quantize_predictions(group, ideal_offset)360 quantization_error += error361 notes_and_rests.append(note_or_rest)362 363 return quantization_error, notes_and_rests364 365 366 best_error = float("inf")367 best_notes_and_rests = None368 best_predictions_per_note = None369 370 for predictions_per_note in range(20, 65, 1):371 for prediction_start_offset in range(predictions_per_note):372 373 error, notes_and_rests = get_quantization_and_error(374 pitch_outputs_and_rests, predictions_per_note,375 prediction_start_offset, ideal_offset)376 377 if error < best_error: 378 best_error = error379 best_notes_and_rests = notes_and_rests380 best_predictions_per_note = predictions_per_note381 382 # At this point, best_notes_and_rests contains the best quantization.383 # Since we don't need to have rests at the beginning, let's remove these:384 #while best_notes_and_rests[0] == 'Rest':385 # best_notes_and_rests = best_notes_and_rests[1:]386 # Also remove silence at the end.387 #while best_notes_and_rests[-1] == 'Rest':388 # best_notes_and_rests = best_notes_and_rests[:-1]389 390 # ____________________________________________________________________________391 # Now let's write the quantized notes as sheet music score!392 # To do it we will use two libraries: [music21](http://web.mit.edu/music21/) and 393 # [Open Sheet Music Display](https://github.com/opensheetmusicdisplay/opensheetmusicdisplay)394 # **Note:** for simplicity, we assume here that all notes have the same duration 395 # (a half note).396 397 # Creating the sheet music score.398 sc = music21.stream.Score()399 # Adjust the speed to match the actual singing.400 bpm = 60 * 60 / best_predictions_per_note401 #print ('bpm: ', bpm)402 a = music21.tempo.MetronomeMark(number=bpm)403 sc.insert(0,a)404 405 for snote in best_notes_and_rests: 406 d = 'half'407 if snote == 'Rest': 408 sc.append(music21.note.Rest(type=d))409 else:410 sc.append(music21.note.Note(snote, type=d))411 412 413 # @title [Run this] Helper function to use Open Sheet Music Display (JS code) 414 # to show a music score415 from IPython.core.display import HTML, Javascript416 from IPython import display417 import json, random418 419 def showScore(score):420 xml = open(score.write('musicxml')).read()421 showMusicXML(xml)422 423 def showMusicXML(xml):424 DIV_ID = "OSMD_div"425 a = display(HTML('<div id="'+DIV_ID+'">loading OpenSheetMusicDisplay</div>'))426 script = """427 var div_id = {{DIV_ID}};428 function loadOSMD() { 429 return new Promise(function(resolve, reject){430 if (window.opensheetmusicdisplay) {431 return resolve(window.opensheetmusicdisplay)432 }433 // OSMD script has a 'define' call which conflicts with requirejs434 var _define = window.define // save the define object 435 window.define = undefined // now the loaded script will ignore requirejs436 var s = document.createElement( 'script' );437 s.setAttribute( 'src', "https://cdn.jsdelivr.net/npm/opensheetmusicdisplay@0.7.6/build/opensheetmusicdisplay.min.js" );438 //s.setAttribute( 'src', "/custom/opensheetmusicdisplay.js" );439 s.onload=function(){440 window.define = _define441 resolve(opensheetmusicdisplay);442 };443 document.body.appendChild( s ); // browser will try to load the new script tag444 }) 445 }446 loadOSMD().then((OSMD)=>{447 window.openSheetMusicDisplay = new OSMD.OpenSheetMusicDisplay(div_id, {448 drawingParameters: "compacttight"449 });450 openSheetMusicDisplay451 .load({{data}})452 .then(453 function() {454 openSheetMusicDisplay.render();455 }456 );457 })458 """.replace('{{DIV_ID}}',DIV_ID).replace('{{data}}',json.dumps(xml))459 display(Javascript(script))460 return a461 462 # rendering the music score463 ###partitura = showScore(sc)464 #print(best_notes_and_rests)465 466 467 468 # ____________________________________________________________________________469 # Let's convert the music notes to a MIDI file and listen to it.470 # To create this file, we can use the stream we created before.471 472 # Saving the recognized musical notes as a MIDI file473 ##converted_audio_file_as_midi = converted_audio_file[:-4] + '.mid'474 ##fp = sc.write('midi', fp=converted_audio_file_as_midi)475 476 ##wav_from_created_midi = converted_audio_file_as_midi.replace(' ', '_') + "_midioutput.wav"477 #print(wav_from_created_midi)478 479 # To listen to it on colab, we need to convert it back to wav. An easy way of 480 # doing that is using Timidity.481 482 #!timidity $converted_audio_file_as_midi -Ow -o $wav_from_created_midi483 return converted_audio_file, fig1, fig2, fig3, fig4,fig5, bpm, best_notes_and_rests#, wav_from_created_midi484 #return converted_audio_file, fig1, fig2, fig3, fig4,fig5, bpm, best_notes_and_rests, partitura, wav_from_created_midi485 486link = "https://www.tensorflow.org/hub/tutorials/spice?hl=es-419&authuser=2"487 488iface = gr.Interface(489 fn=main, 490 title= "Trabajo Práctico N°3 - Detección de tono con SPICE",491 description="Implementación de Modelo con GitHub + Hugging Face🤗-- 🔊✅ " + "Basado en: " + link, 492 inputs = [gr.inputs.Audio(source= "microphone" , type="filepath",label="Ingrese Audio")],493 outputs= [gr.outputs.Audio(label="Audio Original"), 494 gr.outputs.Plot(type="auto",label="Gráfico de Frecuencias"),495 gr.outputs.Plot(type="auto",label="Especto"),496 gr.outputs.Plot(type="auto",label="Pitch Confidence"),497 gr.outputs.Plot(type="auto",label="Notas"),498 gr.outputs.Plot(type="auto",label="Espectro+Notas"),499 gr.outputs.Textbox(label="bpm"),500 gr.outputs.Textbox(label="partitura")],#,501 #gr.outputs.Textbox(type="html",label="partitura1"),502 #gr.outputs.Audio(label="midi")],503 interpretation = "default",504)505 506iface.launch(debug=True)507