thelou1s/TensorflowHubSpice
1
1{2 "cells": [3 {4 "cell_type": "markdown",5 "metadata": {6 "id": "view-in-github",7 "colab_type": "text"8 },9 "source": [10 "<a href=\"https://colab.research.google.com/github/MiguelJ125/SpiceIcaroTP/blob/main/TP3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"11 ]12 },13 {14 "cell_type": "code",15 "execution_count": null,16 "metadata": {17 "colab": {18 "base_uri": "https://localhost:8080/"19 },20 "id": "VIDKxin5EnOn",21 "outputId": "643f3c0e-90ef-4a00-e323-3de746ef2ceb"22 },23 "outputs": [24 {25 "output_type": "stream",26 "name": "stdout",27 "text": [28 "\u001b[K |████████████████████████████████| 2.9 MB 4.9 MB/s \n",29 "\u001b[K |████████████████████████████████| 253 kB 28.0 MB/s \n",30 "\u001b[K |████████████████████████████████| 1.1 MB 35.5 MB/s \n",31 "\u001b[K |████████████████████████████████| 212 kB 38.3 MB/s \n",32 "\u001b[K |████████████████████████████████| 53 kB 904 kB/s \n",33 "\u001b[K |████████████████████████████████| 84 kB 1.3 MB/s \n",34 "\u001b[K |████████████████████████████████| 2.0 MB 59.5 MB/s \n",35 "\u001b[K |████████████████████████████████| 54 kB 2.1 MB/s \n",36 "\u001b[K |████████████████████████████████| 94 kB 2.6 MB/s \n",37 "\u001b[K |████████████████████████████████| 144 kB 55.1 MB/s \n",38 "\u001b[K |████████████████████████████████| 271 kB 54.5 MB/s \n",39 "\u001b[K |████████████████████████████████| 10.9 MB 51.5 MB/s \n",40 "\u001b[K |████████████████████████████████| 58 kB 4.2 MB/s \n",41 "\u001b[K |████████████████████████████████| 79 kB 5.7 MB/s \n",42 "\u001b[K |████████████████████████████████| 43 kB 1.4 MB/s \n",43 "\u001b[K |████████████████████████████████| 4.0 MB 52.0 MB/s \n",44 "\u001b[K |████████████████████████████████| 62 kB 698 kB/s \n",45 "\u001b[K |████████████████████████████████| 856 kB 61.7 MB/s \n",46 "\u001b[K |████████████████████████████████| 58 kB 3.8 MB/s \n",47 "\u001b[?25h Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",48 " Building wheel for python-multipart (setup.py) ... \u001b[?25l\u001b[?25hdone\n"49 ]50 }51 ],52 "source": [53 "# Install Gradio\n",54 "!pip install gradio -q"55 ]56 },57 {58 "cell_type": "code",59 "execution_count": null,60 "metadata": {61 "colab": {62 "base_uri": "https://localhost:8080/"63 },64 "id": "INwxQHDNldmP",65 "outputId": "a49d0bd8-0b20-4981-8d33-2de206e7ad44"66 },67 "outputs": [68 {69 "output_type": "stream",70 "name": "stdout",71 "text": [72 "Reading package lists...\n",73 "Building dependency tree...\n",74 "Reading state information...\n",75 "libsndfile1 is already the newest version (1.0.28-4ubuntu0.18.04.2).\n",76 "The following packages were automatically installed and are no longer required:\n",77 " libnvidia-common-460 nsight-compute-2020.2.0\n",78 "Use 'sudo apt autoremove' to remove them.\n",79 "The following additional packages will be installed:\n",80 " freepats libaudio2 timidity-daemon\n",81 "Suggested packages:\n",82 " nas fluid-soundfont-gm fluid-soundfont-gs pmidi\n",83 "The following NEW packages will be installed:\n",84 " freepats libaudio2 timidity timidity-daemon\n",85 "0 upgraded, 4 newly installed, 0 to remove and 42 not upgraded.\n",86 "Need to get 29.6 MB of archives.\n",87 "After this operation, 35.7 MB of additional disk space will be used.\n",88 "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 freepats all 20060219-1 [29.0 MB]\n",89 "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 libaudio2 amd64 1.9.4-6 [50.3 kB]\n",90 "Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 timidity amd64 2.13.2-41 [585 kB]\n",91 "Get:4 http://archive.ubuntu.com/ubuntu bionic/universe amd64 timidity-daemon all 2.13.2-41 [5,984 B]\n",92 "Fetched 29.6 MB in 1s (29.2 MB/s)\n",93 "debconf: unable to initialize frontend: Dialog\n",94 "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 4.)\n",95 "debconf: falling back to frontend: Readline\n",96 "debconf: unable to initialize frontend: Readline\n",97 "debconf: (This frontend requires a controlling tty.)\n",98 "debconf: falling back to frontend: Teletype\n",99 "dpkg-preconfigure: unable to re-open stdin: \n",100 "Selecting previously unselected package freepats.\n",101 "(Reading database ... 155202 files and directories currently installed.)\n",102 "Preparing to unpack .../freepats_20060219-1_all.deb ...\n",103 "Unpacking freepats (20060219-1) ...\n",104 "Selecting previously unselected package libaudio2:amd64.\n",105 "Preparing to unpack .../libaudio2_1.9.4-6_amd64.deb ...\n",106 "Unpacking libaudio2:amd64 (1.9.4-6) ...\n",107 "Selecting previously unselected package timidity.\n",108 "Preparing to unpack .../timidity_2.13.2-41_amd64.deb ...\n",109 "Unpacking timidity (2.13.2-41) ...\n",110 "Selecting previously unselected package timidity-daemon.\n",111 "Preparing to unpack .../timidity-daemon_2.13.2-41_all.deb ...\n",112 "Unpacking timidity-daemon (2.13.2-41) ...\n",113 "Setting up freepats (20060219-1) ...\n",114 "Setting up libaudio2:amd64 (1.9.4-6) ...\n",115 "Setting up timidity (2.13.2-41) ...\n",116 "Setting up timidity-daemon (2.13.2-41) ...\n",117 "Adding group timidity....done\n",118 "Adding system user timidity....done\n",119 "Adding user `timidity' to group `audio' ...\n",120 "Adding user timidity to group audio\n",121 "Done.\n",122 "invoke-rc.d: could not determine current runlevel\n",123 "invoke-rc.d: policy-rc.d denied execution of stop.\n",124 "invoke-rc.d: could not determine current runlevel\n",125 "invoke-rc.d: policy-rc.d denied execution of start.\n",126 "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",127 "Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n",128 "/sbin/ldconfig.real: /usr/local/lib/python3.7/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",129 "\n",130 "Processing triggers for systemd (237-3ubuntu10.53) ...\n"131 ]132 }133 ],134 "source": [135 "# Install timidy\n",136 "!sudo apt-get install -q -y timidity libsndfile1"137 ]138 },139 {140 "cell_type": "code",141 "execution_count": null,142 "metadata": {143 "colab": {144 "base_uri": "https://localhost:8080/"145 },146 "id": "7zlt1-FVluYE",147 "outputId": "ee460fc0-2778-4db9-b13e-4b9c39a2e297"148 },149 "outputs": [150 {151 "output_type": "stream",152 "name": "stdout",153 "text": [154 "Requirement already satisfied: pydub in /usr/local/lib/python3.7/dist-packages (0.25.1)\n",155 "Collecting numba==0.48\n",156 " Downloading numba-0.48.0-1-cp37-cp37m-manylinux2014_x86_64.whl (3.5 MB)\n",157 "\u001b[K |████████████████████████████████| 3.5 MB 4.9 MB/s \n",158 "\u001b[?25hRequirement already satisfied: librosa in /usr/local/lib/python3.7/dist-packages (0.8.1)\n",159 "Requirement already satisfied: music21 in /usr/local/lib/python3.7/dist-packages (5.5.0)\n",160 "Collecting llvmlite<0.32.0,>=0.31.0dev0\n",161 " Downloading llvmlite-0.31.0-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)\n",162 "\u001b[K |████████████████████████████████| 20.2 MB 65.7 MB/s \n",163 "\u001b[?25hRequirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from numba==0.48) (1.21.6)\n",164 "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba==0.48) (57.4.0)\n",165 "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from librosa) (1.4.1)\n",166 "Requirement already satisfied: resampy>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from librosa) (0.2.2)\n",167 "Requirement already satisfied: pooch>=1.0 in /usr/local/lib/python3.7/dist-packages (from librosa) (1.6.0)\n",168 "Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from librosa) (1.0.2)\n",169 "Requirement already satisfied: soundfile>=0.10.2 in /usr/local/lib/python3.7/dist-packages (from librosa) (0.10.3.post1)\n",170 "Requirement already satisfied: decorator>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from librosa) (4.4.2)\n",171 "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from librosa) (21.3)\n",172 "Requirement already satisfied: audioread>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from librosa) (2.1.9)\n",173 "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.7/dist-packages (from librosa) (1.1.0)\n",174 "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->librosa) (3.0.8)\n",175 "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from pooch>=1.0->librosa) (2.23.0)\n",176 "Requirement already satisfied: appdirs>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from pooch>=1.0->librosa) (1.4.4)\n",177 "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2.10)\n",178 "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pooch>=1.0->librosa) (1.24.3)\n",179 "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2021.10.8)\n",180 "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->pooch>=1.0->librosa) (3.0.4)\n",181 "Requirement already satisfied: six>=1.3 in /usr/local/lib/python3.7/dist-packages (from resampy>=0.2.2->librosa) (1.15.0)\n",182 "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn!=0.19.0,>=0.14.0->librosa) (3.1.0)\n",183 "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.7/dist-packages (from soundfile>=0.10.2->librosa) (1.15.0)\n",184 "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.0->soundfile>=0.10.2->librosa) (2.21)\n",185 "Installing collected packages: llvmlite, numba\n",186 " Attempting uninstall: llvmlite\n",187 " Found existing installation: llvmlite 0.34.0\n",188 " Uninstalling llvmlite-0.34.0:\n",189 " Successfully uninstalled llvmlite-0.34.0\n",190 " Attempting uninstall: numba\n",191 " Found existing installation: numba 0.51.2\n",192 " Uninstalling numba-0.51.2:\n",193 " Successfully uninstalled numba-0.51.2\n",194 "Successfully installed llvmlite-0.31.0 numba-0.48.0\n"195 ]196 }197 ],198 "source": [199 "# All the imports to deal with sound data\n",200 "!pip install pydub numba==0.48 librosa music21"201 ]202 },203 {204 "cell_type": "code",205 "execution_count": null,206 "metadata": {207 "id": "VAKjZZJDl7mO",208 "colab": {209 "base_uri": "https://localhost:8080/"210 },211 "outputId": "6074e108-983c-4085-e533-6c715fd680b2"212 },213 "outputs": [214 {215 "output_type": "stream",216 "name": "stderr",217 "text": [218 "/usr/local/lib/python3.7/dist-packages/paramiko/transport.py:236: CryptographyDeprecationWarning: Blowfish has been deprecated\n",219 " \"class\": algorithms.Blowfish,\n"220 ]221 }222 ],223 "source": [224 "# Import Libraries\n",225 "\n",226 "import gradio as gr\n",227 "import time\n",228 "\n",229 "import tensorflow as tf\n",230 "import tensorflow_hub as hub\n",231 "\n",232 "import numpy as np\n",233 "import matplotlib.pyplot as plt\n",234 "import librosa\n",235 "from librosa import display as librosadisplay\n",236 "\n",237 "import logging\n",238 "import math\n",239 "import statistics\n",240 "import sys\n",241 "\n",242 "from IPython.display import Audio, Javascript\n",243 "from scipy.io import wavfile\n",244 "\n",245 "from base64 import b64decode\n",246 "\n",247 "import music21\n",248 "from pydub import AudioSegment\n",249 "\n",250 "logger = logging.getLogger()\n",251 "logger.setLevel(logging.ERROR)\n",252 "\n",253 "#print(\"tensorflow: %s\" % tf.__version__)\n",254 "#print(\"librosa: %s\" % librosa.__version__)"255 ]256 },257 {258 "cell_type": "code",259 "execution_count": null,260 "metadata": {261 "id": "NYRdWK1_Na6q"262 },263 "outputs": [],264 "source": [265 "# The audio input file\n",266 "# Now the hardest part: Record your singing! :)\n",267 "\n",268 "# We provide four methods to obtain an audio file:\n",269 "\n",270 "# 1. Record audio directly in Gradio\n",271 "# 2. Use a file saved on Google Drive"272 ]273 },274 {275 "cell_type": "code",276 "execution_count": null,277 "metadata": {278 "id": "x0eCz-EAiqvJ"279 },280 "outputs": [],281 "source": [282 "# Use a file saved on Google Drive\n",283 "INPUT_SOURCE = 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav'"284 ]285 },286 {287 "cell_type": "code",288 "execution_count": null,289 "metadata": {290 "colab": {291 "base_uri": "https://localhost:8080/"292 },293 "id": "DG6JJCFgNpnF",294 "outputId": "98392a37-bb92-4a7d-c049-256364bd0ad5"295 },296 "outputs": [297 {298 "output_type": "stream",299 "name": "stdout",300 "text": [301 "--2022-05-03 23:44:36-- https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav\n",302 "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.204.128, 108.177.11.128, 172.253.123.128, ...\n",303 "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.204.128|:443... connected.\n",304 "HTTP request sent, awaiting response... 200 OK\n",305 "Length: 384728 (376K) [audio/wav]\n",306 "Saving to: ‘c-scale.wav’\n",307 "\n",308 "\rc-scale.wav 0%[ ] 0 --.-KB/s \rc-scale.wav 100%[===================>] 375.71K --.-KB/s in 0.005s \n",309 "\n",310 "2022-05-03 23:44:36 (73.4 MB/s) - ‘c-scale.wav’ saved [384728/384728]\n",311 "\n"312 ]313 }314 ],315 "source": [316 "!wget --no-check-certificate 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav' -O c-scale.wav"317 ]318 },319 {320 "cell_type": "code",321 "execution_count": null,322 "metadata": {323 "id": "hlVhhsruNwfX"324 },325 "outputs": [],326 "source": [327 "uploaded_file_name = 'c-scale.wav'"328 ]329 },330 {331 "cell_type": "code",332 "execution_count": null,333 "metadata": {334 "colab": {335 "base_uri": "https://localhost:8080/",336 "height": 35337 },338 "id": "_yc2tLWBYu4w",339 "outputId": "f9e2dab8-9d63-4e38-e200-42db26479ad8"340 },341 "outputs": [342 {343 "output_type": "execute_result",344 "data": {345 "text/plain": [346 "'c-scale.wav'"347 ],348 "application/vnd.google.colaboratory.intrinsic+json": {349 "type": "string"350 }351 },352 "metadata": {},353 "execution_count": 9354 }355 ],356 "source": [357 "uploaded_file_name"358 ]359 },360 {361 "cell_type": "code",362 "execution_count": null,363 "metadata": {364 "id": "rQaBdWLMgcXh"365 },366 "outputs": [],367 "source": [368 "# Function that converts the user-created audio to the format that the model \n",369 "# expects: bitrate 16kHz and only one channel (mono).\n",370 "\n",371 "EXPECTED_SAMPLE_RATE = 16000\n",372 "\n",373 "def convert_audio_for_model(user_file, output_file='converted_audio_file.wav'):\n",374 " audio = AudioSegment.from_file(user_file)\n",375 " audio = audio.set_frame_rate(EXPECTED_SAMPLE_RATE).set_channels(1)\n",376 " audio.export(output_file, format=\"wav\")\n",377 " return output_file"378 ]379 },380 {381 "cell_type": "code",382 "execution_count": null,383 "metadata": {384 "id": "TA09s5XshFXO"385 },386 "outputs": [],387 "source": [388 "MAX_ABS_INT16 = 32768.0\n",389 "\n",390 "def plot_stft(x, sample_rate, show_black_and_white=False):\n",391 " x_stft = np.abs(librosa.stft(x, n_fft=2048))\n",392 " fig, ax = plt.subplots()\n",393 " fig.set_size_inches(20, 10)\n",394 " x_stft_db = librosa.amplitude_to_db(x_stft, ref=np.max)\n",395 "\n",396 " if(show_black_and_white):\n",397 " librosadisplay.specshow(data=x_stft_db, \n",398 " y_axis='log', \n",399 " sr=sample_rate, \n",400 " cmap='gray_r')\n",401 " else:\n",402 " librosadisplay.specshow(data=x_stft_db, \n",403 " y_axis='log', \n",404 " sr=sample_rate)\n",405 "\n",406 " plt.colorbar(format='%+2.0f dB')\n",407 "\n",408 " return fig"409 ]410 },411 {412 "cell_type": "code",413 "execution_count": null,414 "metadata": {415 "colab": {416 "base_uri": "https://localhost:8080/"417 },418 "id": "R85QA7Qgt9Fv",419 "outputId": "f96e3330-b174-4b1c-b514-462f94abcc1c"420 },421 "outputs": [422 {423 "output_type": "stream",424 "name": "stdout",425 "text": [426 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"427 ]428 },429 {430 "output_type": "stream",431 "name": "stderr",432 "text": [433 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"434 ]435 },436 {437 "output_type": "stream",438 "name": "stdout",439 "text": [440 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"441 ]442 },443 {444 "output_type": "stream",445 "name": "stderr",446 "text": [447 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"448 ]449 },450 {451 "output_type": "stream",452 "name": "stdout",453 "text": [454 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"455 ]456 },457 {458 "output_type": "stream",459 "name": "stderr",460 "text": [461 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"462 ]463 },464 {465 "output_type": "stream",466 "name": "stdout",467 "text": [468 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"469 ]470 },471 {472 "output_type": "stream",473 "name": "stderr",474 "text": [475 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"476 ]477 },478 {479 "output_type": "stream",480 "name": "stdout",481 "text": [482 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"483 ]484 },485 {486 "output_type": "stream",487 "name": "stderr",488 "text": [489 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"490 ]491 },492 {493 "output_type": "stream",494 "name": "stdout",495 "text": [496 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"497 ]498 },499 {500 "output_type": "stream",501 "name": "stderr",502 "text": [503 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"504 ]505 },506 {507 "output_type": "stream",508 "name": "stdout",509 "text": [510 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"511 ]512 },513 {514 "output_type": "stream",515 "name": "stderr",516 "text": [517 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"518 ]519 },520 {521 "output_type": "stream",522 "name": "stdout",523 "text": [524 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"525 ]526 },527 {528 "output_type": "stream",529 "name": "stderr",530 "text": [531 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"532 ]533 },534 {535 "output_type": "stream",536 "name": "stdout",537 "text": [538 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"539 ]540 },541 {542 "output_type": "stream",543 "name": "stderr",544 "text": [545 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"546 ]547 },548 {549 "output_type": "stream",550 "name": "stdout",551 "text": [552 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"553 ]554 },555 {556 "output_type": "stream",557 "name": "stderr",558 "text": [559 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"560 ]561 },562 {563 "output_type": "stream",564 "name": "stdout",565 "text": [566 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"567 ]568 },569 {570 "output_type": "stream",571 "name": "stderr",572 "text": [573 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"574 ]575 },576 {577 "output_type": "stream",578 "name": "stdout",579 "text": [580 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"581 ]582 },583 {584 "output_type": "stream",585 "name": "stderr",586 "text": [587 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"588 ]589 },590 {591 "output_type": "stream",592 "name": "stdout",593 "text": [594 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"595 ]596 },597 {598 "output_type": "stream",599 "name": "stderr",600 "text": [601 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"602 ]603 },604 {605 "output_type": "stream",606 "name": "stdout",607 "text": [608 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"609 ]610 },611 {612 "output_type": "stream",613 "name": "stderr",614 "text": [615 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"616 ]617 },618 {619 "output_type": "stream",620 "name": "stdout",621 "text": [622 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"623 ]624 },625 {626 "output_type": "stream",627 "name": "stderr",628 "text": [629 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"630 ]631 },632 {633 "output_type": "stream",634 "name": "stdout",635 "text": [636 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"637 ]638 },639 {640 "output_type": "stream",641 "name": "stderr",642 "text": [643 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'global_step:0' shape=() dtype=int64_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"644 ]645 },646 {647 "output_type": "stream",648 "name": "stdout",649 "text": [650 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"651 ]652 },653 {654 "output_type": "stream",655 "name": "stderr",656 "text": [657 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/conv2d/kernel:0' shape=(1, 3, 1, 64) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"658 ]659 },660 {661 "output_type": "stream",662 "name": "stdout",663 "text": [664 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"665 ]666 },667 {668 "output_type": "stream",669 "name": "stderr",670 "text": [671 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/gamma:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"672 ]673 },674 {675 "output_type": "stream",676 "name": "stdout",677 "text": [678 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"679 ]680 },681 {682 "output_type": "stream",683 "name": "stderr",684 "text": [685 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/beta:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"686 ]687 },688 {689 "output_type": "stream",690 "name": "stdout",691 "text": [692 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"693 ]694 },695 {696 "output_type": "stream",697 "name": "stderr",698 "text": [699 "WARNING:tensorflow:Unable to create a python object for variable <tf.Variable 'encoder/batch_normalization/moving_mean:0' shape=(64,) dtype=float32_ref> because it is a reference variable. It may not be visible to training APIs. If this is a problem, consider rebuilding the SavedModel after running tf.compat.v1.enable_resource_variables().\n"700 ]701 }702 ],703 "source": [704 "# Executing the Model\n",705 "# Loading the SPICE model is easy:\n",706 "model = hub.load(\"https://tfhub.dev/google/spice/2\")"707 ]708 },709 {710 "cell_type": "code",711 "execution_count": null,712 "metadata": {713 "id": "v-DwOV31urI8"714 },715 "outputs": [],716 "source": [717 "def plot_pitch_conf(pitch_outputs,confidence_outputs):\n",718 " fig, ax = plt.subplots()\n",719 " fig.set_size_inches(20, 10)\n",720 " plt.plot(pitch_outputs, label='pitch')\n",721 " plt.plot(confidence_outputs, label='confidence')\n",722 " plt.legend(loc=\"lower right\")\n",723 " return fig"724 ]725 },726 {727 "cell_type": "code",728 "execution_count": null,729 "metadata": {730 "id": "93a_LcFsP6yp"731 },732 "outputs": [],733 "source": [734 "def plot_pitch_conf_notes(confident_pitch_outputs_x,confident_pitch_outputs_y):\n",735 " fig, ax = plt.subplots()\n",736 " fig.set_size_inches(20, 10)\n",737 " ax.set_ylim([0, 1])\n",738 " plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, )\n",739 " plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, c=\"r\")\n",740 " return fig"741 ]742 },743 {744 "cell_type": "code",745 "execution_count": null,746 "metadata": {747 "id": "gttS-IZfQ8bt"748 },749 "outputs": [],750 "source": [751 "def output2hz(pitch_output):\n",752 " # Constants taken from https://tfhub.dev/google/spice/2\n",753 " PT_OFFSET = 25.58\n",754 " PT_SLOPE = 63.07\n",755 " FMIN = 10.0;\n",756 " BINS_PER_OCTAVE = 12.0;\n",757 " cqt_bin = pitch_output * PT_SLOPE + PT_OFFSET;\n",758 " return FMIN * 2.0 ** (1.0 * cqt_bin / BINS_PER_OCTAVE)"759 ]760 },761 {762 "cell_type": "code",763 "execution_count": null,764 "metadata": {765 "id": "SvKkwCWdRWPj"766 },767 "outputs": [],768 "source": [769 "def espectro_notas(audio_samples,EXPECTED_SAMPLE_RATE,confident_pitch_outputs_x,confident_pitch_values_hz):\n",770 " fig, ax = plt.subplots()\n",771 " plot_stft(audio_samples / MAX_ABS_INT16 , \n",772 " sample_rate=EXPECTED_SAMPLE_RATE, show_black_and_white=True)\n",773 " # Note: conveniently, since the plot is in log scale, the pitch outputs \n",774 " # also get converted to the log scale automatically by matplotlib.\n",775 " plt.scatter(confident_pitch_outputs_x, confident_pitch_values_hz, c=\"r\")\n",776 " return fig"777 ]778 },779 {780 "cell_type": "code",781 "execution_count": null,782 "metadata": {783 "id": "qv-HNpGWBHjt"784 },785 "outputs": [],786 "source": [787 " def hz2offset(freq):\n",788 " # This measures the quantization error for a single note.\n",789 " if freq == 0: # Rests always have zero error.\n",790 " return None\n",791 " # Quantized note.\n",792 " h = round(12 * math.log2(freq / C0))\n",793 " return 12 * math.log2(freq / C0) - h"794 ]795 },796 {797 "cell_type": "code",798 "execution_count": null,799 "metadata": {800 "id": "wZ9snfQxBTK4"801 },802 "outputs": [],803 "source": [804 "def quantize_predictions(group, ideal_offset):\n",805 " # Group values are either 0, or a pitch in Hz.\n",806 " non_zero_values = [v for v in group if v != 0]\n",807 " zero_values_count = len(group) - len(non_zero_values)\n",808 "\n",809 " # Create a rest if 80% is silent, otherwise create a note.\n",810 " if zero_values_count > 0.8 * len(group):\n",811 " # Interpret as a rest. Count each dropped note as an error, weighted a bit\n",812 " # worse than a badly sung note (which would 'cost' 0.5).\n",813 " return 0.51 * len(non_zero_values), \"Rest\"\n",814 " else:\n",815 " # Interpret as note, estimating as mean of non-rest predictions.\n",816 " h = round(\n",817 " statistics.mean([\n",818 " 12 * math.log2(freq / C0) - ideal_offset for freq in non_zero_values\n",819 " ]))\n",820 " octave = h // 12\n",821 " n = h % 12\n",822 " note = note_names[n] + str(octave)\n",823 " # Quantization error is the total difference from the quantized note.\n",824 " error = sum([\n",825 " abs(12 * math.log2(freq / C0) - ideal_offset - h)\n",826 " for freq in non_zero_values\n",827 " ])\n",828 " return error, note"829 ]830 },831 {832 "cell_type": "code",833 "execution_count": null,834 "metadata": {835 "id": "YK3tLfC0BYhj"836 },837 "outputs": [],838 "source": [839 "def get_quantization_and_error(pitch_outputs_and_rests, predictions_per_eighth,\n",840 " prediction_start_offset, ideal_offset):\n",841 " # Apply the start offset - we can just add the offset as rests.\n",842 " pitch_outputs_and_rests = [0] * prediction_start_offset + \\\n",843 " pitch_outputs_and_rests\n",844 " # Collect the predictions for each note (or rest).\n",845 " groups = [\n",846 " pitch_outputs_and_rests[i:i + predictions_per_eighth]\n",847 " for i in range(0, len(pitch_outputs_and_rests), predictions_per_eighth)\n",848 " ]\n",849 "\n",850 " quantization_error = 0\n",851 "\n",852 " notes_and_rests = []\n",853 " for group in groups:\n",854 " error, note_or_rest = quantize_predictions(group, ideal_offset)\n",855 " quantization_error += error\n",856 " notes_and_rests.append(note_or_rest)\n",857 "\n",858 " return quantization_error, notes_and_rests"859 ]860 },861 {862 "cell_type": "code",863 "execution_count": null,864 "metadata": {865 "id": "vayQfo7EeeL7"866 },867 "outputs": [],868 "source": [869 "def main(audio):\n",870 "\n",871 " # Preparing the audio data\n",872 " # Now we have the audio, let's convert it to the expected format and then \n",873 " # listen to it!\n",874 " # The SPICE model needs as input an audio file at a sampling rate of 16kHz and\n",875 " # with only one channel (mono). \n",876 " # To help you with this part, we created a function(`convert_audio_for_model`) \n",877 " #to convert any wav file you have to the model's expected format:\n",878 "\n",879 "\n",880 " # Converting to the expected format for the model\n",881 " # in all the input 4 input method before, the uploaded file name is at\n",882 " # the variable uploaded_file_name\n",883 " converted_audio_file = convert_audio_for_model(audio)\n",884 "\n",885 " # Loading audio samples from the wav file:\n",886 " sample_rate, audio_samples = wavfile.read(converted_audio_file, 'rb')\n",887 "\n",888 " audio_samples = audio_samples / float(MAX_ABS_INT16)\n",889 "\n",890 "\n",891 " # We now feed the audio to the SPICE tf.hub model to obtain pitch and uncertainty outputs as tensors.\n",892 " model_output = model.signatures[\"serving_default\"](tf.constant(audio_samples, tf.float32))\n",893 "\n",894 " pitch_outputs = model_output[\"pitch\"]\n",895 " uncertainty_outputs = model_output[\"uncertainty\"]\n",896 "\n",897 " # 'Uncertainty' basically means the inverse of confidence.\n",898 " confidence_outputs = 1.0 - uncertainty_outputs\n",899 " \n",900 "\n",901 " confidence_outputs = list(confidence_outputs)\n",902 " pitch_outputs = [ float(x) for x in pitch_outputs]\n",903 "\n",904 " indices = range(len (pitch_outputs))\n",905 " confident_pitch_outputs = [ (i,p) \n",906 " for i, p, c in zip(indices, pitch_outputs, confidence_outputs) if c >= 0.9 ]\n",907 " confident_pitch_outputs_x, confident_pitch_outputs_y = zip(*confident_pitch_outputs)\n",908 "\n",909 " confident_pitch_values_hz = [ output2hz(p) for p in confident_pitch_outputs_y ]\n",910 " \n",911 "\n",912 " #Plot waves\n",913 " fig1 = plt.figure()\n",914 " plt.plot(audio_samples)\n",915 "\n",916 " #Plot \n",917 " fig2 = plot_stft(audio_samples / MAX_ABS_INT16 , sample_rate=EXPECTED_SAMPLE_RATE)\n",918 "\n",919 " #Plot Pitch & Confidence\n",920 " fig3 = plot_pitch_conf(pitch_outputs,confidence_outputs)\n",921 "\n",922 " \n",923 " #Plot Pitch & Confidence Notes\n",924 " fig4 = plot_pitch_conf_notes(confident_pitch_outputs_x,confident_pitch_outputs_y)\n",925 "\n",926 " #Plot Espectro + Notes\n",927 " #fig5 = espectro_notas(audio_samples,EXPECTED_SAMPLE_RATE,confident_pitch_outputs_x,confident_pitch_values_hz)\n",928 "\n",929 " ####### JC #######\n",930 " x = audio_samples / MAX_ABS_INT16\n",931 " sample_rate = EXPECTED_SAMPLE_RATE\n",932 " show_black_and_white=True\n",933 "\n",934 " x_stft = np.abs(librosa.stft(x, n_fft=2048))\n",935 " fig5, ax3 = plt.subplots()\n",936 " fig5.set_size_inches(20, 10)\n",937 " x_stft_db = librosa.amplitude_to_db(x_stft, ref=np.max)\n",938 " if(show_black_and_white):\n",939 " librosadisplay.specshow(data=x_stft_db, y_axis='log', \n",940 " sr=sample_rate, cmap='gray_r')\n",941 " else:\n",942 " librosadisplay.specshow(data=x_stft_db, y_axis='log', sr=sample_rate)\n",943 "\n",944 " confident_pitch_values_hz = [ output2hz(p) for p in confident_pitch_outputs_y ] \n",945 " plt.scatter(confident_pitch_outputs_x, confident_pitch_values_hz, c=\"r\")\n",946 " \n",947 "\n",948 " ###### FIN CAMBIO JC ########\n",949 "\n",950 "\n",951 " # ############################################################################\n",952 " # Converting to musical notes ################################################\n",953 "\n",954 " # Now that we have the pitch values, let's convert them to notes!\n",955 " # This is part is challenging by itself. We have to take into account two \n",956 " # things:\n",957 " # 1. the rests (when there's no singing) \n",958 " # 2. the size of each note (offsets) \n",959 "\n",960 " # ----------------------------------------------------------------------------\n",961 " ### 1: Adding zeros to the output to indicate when there's no singing\n",962 "\n",963 " pitch_outputs_and_rests = [\n",964 " output2hz(p) if c >= 0.9 else 0\n",965 " for i, p, c in zip(indices, pitch_outputs, confidence_outputs)\n",966 " ]\n",967 "\n",968 " # ----------------------------------------------------------------------------\n",969 " ### 2: Adding note offsets\n",970 " # When a person sings freely, the melody may have an offset to the absolute \n",971 " # pitch values that notes can represent.\n",972 " # Hence, to convert predictions to notes, one needs to correct for this \n",973 " # possible offset.\n",974 " # This is what the following code computes.\n",975 "\n",976 " A4 = 440\n",977 " C0 = A4 * pow(2, -4.75)\n",978 " note_names = [\"C\", \"C#\", \"D\", \"D#\", \"E\", \"F\", \"F#\", \"G\", \"G#\", \"A\", \"A#\", \"B\"]\n",979 "\n",980 " def hz2offset(freq):\n",981 " # This measures the quantization error for a single note.\n",982 " if freq == 0: # Rests always have zero error.\n",983 " return None\n",984 " # Quantized note.\n",985 " h = round(12 * math.log2(freq / C0))\n",986 " return 12 * math.log2(freq / C0) - h\n",987 "\n",988 "\n",989 " # The ideal offset is the mean quantization error for all the notes\n",990 " # (excluding rests):\n",991 " offsets = [hz2offset(p) for p in pitch_outputs_and_rests if p != 0]\n",992 " #print(\"offsets: \", offsets)\n",993 " off = offsets\n",994 "\n",995 " ideal_offset = statistics.mean(offsets)\n",996 " #print(\"ideal offset: \", ideal_offset)\n",997 " ideal_off = ideal_offset\n",998 "\n",999 " # We can now use some heuristics to try and estimate the most likely sequence \n",1000 " # of notes that were sung.\n",1001 " # The ideal offset computed above is one ingredient - but we also need to know \n",1002 " # the speed (how many predictions make, say, an eighth?), and the time offset \n",1003 " # to start quantizing. To keep it simple, we'll just try different speeds and \n",1004 " # time offsets and measure the quantization error, using in the end the values \n",1005 " # that minimize this error.\n",1006 "\n",1007 " def quantize_predictions(group, ideal_offset):\n",1008 " # Group values are either 0, or a pitch in Hz.\n",1009 " non_zero_values = [v for v in group if v != 0]\n",1010 " zero_values_count = len(group) - len(non_zero_values)\n",1011 "\n",1012 " # Create a rest if 80% is silent, otherwise create a note.\n",1013 " if zero_values_count > 0.8 * len(group):\n",1014 " # Interpret as a rest. Count each dropped note as an error, weighted a bit\n",1015 " # worse than a badly sung note (which would 'cost' 0.5).\n",1016 " return 0.51 * len(non_zero_values), \"Rest\"\n",1017 " else:\n",1018 " # Interpret as note, estimating as mean of non-rest predictions.\n",1019 " h = round(\n",1020 " statistics.mean([\n",1021 " 12 * math.log2(freq / C0) - ideal_offset for freq in non_zero_values\n",1022 " ]))\n",1023 " octave = h // 12\n",1024 " n = h % 12\n",1025 " note = note_names[n] + str(octave)\n",1026 " # Quantization error is the total difference from the quantized note.\n",1027 " error = sum([\n",1028 " abs(12 * math.log2(freq / C0) - ideal_offset - h)\n",1029 " for freq in non_zero_values\n",1030 " ])\n",1031 " return error, note\n",1032 "\n",1033 "\n",1034 " def get_quantization_and_error(pitch_outputs_and_rests, predictions_per_eighth,\n",1035 " prediction_start_offset, ideal_offset):\n",1036 " # Apply the start offset - we can just add the offset as rests.\n",1037 " pitch_outputs_and_rests = [0] * prediction_start_offset + \\\n",1038 " pitch_outputs_and_rests\n",1039 " # Collect the predictions for each note (or rest).\n",1040 " groups = [\n",1041 " pitch_outputs_and_rests[i:i + predictions_per_eighth]\n",1042 " for i in range(0, len(pitch_outputs_and_rests), predictions_per_eighth)\n",1043 " ]\n",1044 "\n",1045 " quantization_error = 0\n",1046 "\n",1047 " notes_and_rests = []\n",1048 " for group in groups:\n",1049 " error, note_or_rest = quantize_predictions(group, ideal_offset)\n",1050 " quantization_error += error\n",1051 " notes_and_rests.append(note_or_rest)\n",1052 "\n",1053 " return quantization_error, notes_and_rests\n",1054 "\n",1055 "\n",1056 " best_error = float(\"inf\")\n",1057 " best_notes_and_rests = None\n",1058 " best_predictions_per_note = None\n",1059 "\n",1060 " for predictions_per_note in range(20, 65, 1):\n",1061 " for prediction_start_offset in range(predictions_per_note):\n",1062 "\n",1063 " error, notes_and_rests = get_quantization_and_error(\n",1064 " pitch_outputs_and_rests, predictions_per_note,\n",1065 " prediction_start_offset, ideal_offset)\n",1066 "\n",1067 " if error < best_error: \n",1068 " best_error = error\n",1069 " best_notes_and_rests = notes_and_rests\n",1070 " best_predictions_per_note = predictions_per_note\n",1071 "\n",1072 " # At this point, best_notes_and_rests contains the best quantization.\n",1073 " # Since we don't need to have rests at the beginning, let's remove these:\n",1074 " while best_notes_and_rests[0] == 'Rest':\n",1075 " best_notes_and_rests = best_notes_and_rests[1:]\n",1076 " # Also remove silence at the end.\n",1077 " while best_notes_and_rests[-1] == 'Rest':\n",1078 " best_notes_and_rests = best_notes_and_rests[:-1]\n",1079 " \n",1080 " # ____________________________________________________________________________\n",1081 " # Now let's write the quantized notes as sheet music score!\n",1082 " # To do it we will use two libraries: [music21](http://web.mit.edu/music21/) and \n",1083 " # [Open Sheet Music Display](https://github.com/opensheetmusicdisplay/opensheetmusicdisplay)\n",1084 " # **Note:** for simplicity, we assume here that all notes have the same duration \n",1085 " # (a half note).\n",1086 "\n",1087 " # Creating the sheet music score.\n",1088 " sc = music21.stream.Score()\n",1089 " # Adjust the speed to match the actual singing.\n",1090 " bpm = 60 * 60 / best_predictions_per_note\n",1091 " #print ('bpm: ', bpm)\n",1092 " a = music21.tempo.MetronomeMark(number=bpm)\n",1093 " sc.insert(0,a)\n",1094 "\n",1095 " for snote in best_notes_and_rests: \n",1096 " d = 'half'\n",1097 " if snote == 'Rest': \n",1098 " sc.append(music21.note.Rest(type=d))\n",1099 " else:\n",1100 " sc.append(music21.note.Note(snote, type=d))\n",1101 "\n",1102 " # to show a music score\n",1103 " from IPython.core.display import display, HTML, Javascript\n",1104 " import json, random\n",1105 "\n",1106 " def showScore(score):\n",1107 " xml = open(score.write('musicxml')).read()\n",1108 " showMusicXML(xml)\n",1109 " \n",1110 " def showMusicXML(xml):\n",1111 " DIV_ID = \"OSMD_div\"\n",1112 " a = display(HTML('<div id=\"'+DIV_ID+'\">loading OpenSheetMusicDisplay</div>'))\n",1113 " script = \"\"\"\n",1114 " var div_id = {{DIV_ID}};\n",1115 " function loadOSMD() { \n",1116 " return new Promise(function(resolve, reject){\n",1117 " if (window.opensheetmusicdisplay) {\n",1118 " return resolve(window.opensheetmusicdisplay)\n",1119 " }\n",1120 " // OSMD script has a 'define' call which conflicts with requirejs\n",1121 " var _define = window.define // save the define object \n",1122 " window.define = undefined // now the loaded script will ignore requirejs\n",1123 " var s = document.createElement( 'script' );\n",1124 " s.setAttribute( 'src', \"https://cdn.jsdelivr.net/npm/opensheetmusicdisplay@0.7.6/build/opensheetmusicdisplay.min.js\" );\n",1125 " //s.setAttribute( 'src', \"/custom/opensheetmusicdisplay.js\" );\n",1126 " s.onload=function(){\n",1127 " window.define = _define\n",1128 " resolve(opensheetmusicdisplay);\n",1129 " };\n",1130 " document.body.appendChild( s ); // browser will try to load the new script tag\n",1131 " }) \n",1132 " }\n",1133 " loadOSMD().then((OSMD)=>{\n",1134 " window.openSheetMusicDisplay = new OSMD.OpenSheetMusicDisplay(div_id, {\n",1135 " drawingParameters: \"compacttight\"\n",1136 " });\n",1137 " openSheetMusicDisplay\n",1138 " .load({{data}})\n",1139 " .then(\n",1140 " function() {\n",1141 " openSheetMusicDisplay.render();\n",1142 " }\n",1143 " );\n",1144 " })\n",1145 " \"\"\".replace('{{DIV_ID}}',DIV_ID).replace('{{data}}',json.dumps(xml))\n",1146 " #display(Javascript(script))\n",1147 " return a\n",1148 "\n",1149 " # rendering the music score\n",1150 " partitura = showScore(sc)\n",1151 " #print(best_notes_and_rests)\n",1152 "\n",1153 "\n",1154 "\n",1155 " # ____________________________________________________________________________\n",1156 " # Let's convert the music notes to a MIDI file and listen to it.\n",1157 " # To create this file, we can use the stream we created before.\n",1158 "\n",1159 " # Saving the recognized musical notes as a MIDI file\n",1160 " converted_audio_file_as_midi = converted_audio_file[:-4] + '.mid'\n",1161 " fp = sc.write('midi', fp=converted_audio_file_as_midi)\n",1162 "\n",1163 " wav_from_created_midi = converted_audio_file_as_midi.replace(' ', '_') + \"_midioutput.wav\"\n",1164 " #print(wav_from_created_midi)\n",1165 "\n",1166 " # To listen to it on colab, we need to convert it back to wav. An easy way of \n",1167 " # doing that is using Timidity.\n",1168 "\n",1169 " !timidity $converted_audio_file_as_midi -Ow -o $wav_from_created_midi\n",1170 "\n",1171 " return converted_audio_file, fig1, fig2, fig3, fig4,fig5, bpm, best_notes_and_rests, partitura, wav_from_created_midi"1172 ]1173 },1174 {1175 "cell_type": "code",1176 "source": [1177 "link = \"https://www.tensorflow.org/hub/tutorials/spice?hl=es-419&authuser=2\""1178 ],1179 "metadata": {1180 "id": "5lu2GGYRO_3g"1181 },1182 "execution_count": null,1183 "outputs": []1184 },1185 {1186 "cell_type": "code",1187 "execution_count": null,1188 "metadata": {1189 "colab": {1190 "base_uri": "https://localhost:8080/",1191 "height": 10001192 },1193 "id": "0C2k-VhuN_wG",1194 "outputId": "6a116ff8-b95e-4e0b-f7a3-a3edf14b4639"1195 },1196 "outputs": [1197 {1198 "output_type": "stream",1199 "name": "stdout",1200 "text": [