cscan/Audio_to_text_classification

1import os2os.system("pip install git+https://github.com/openai/whisper.git")3import gradio as gr4import whisper5from huggingface_hub import from_pretrained_keras6from transformers import AutoTokenizer, AutoModelForSequenceClassification7from transformers import pipeline8from sklearn.preprocessing import StandardScaler9import logging10import librosa11import numpy as np12import pickle13 14 15 16#call tokenizer and NLP model for text classification17tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")18model_nlp = AutoModelForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")19 20 21# call whisper model for audio/speech processing22model = whisper.load_model("small")23 24# call model for audio emotions25reloaded_model = from_pretrained_keras('jmparejaz/RAVDESS-CREMAD_AudioEmotionClassifier')26 27# call scaler and decoder28with open("scaler.pkl", "rb") as f:29    scaler = pickle.load(f)30 31with open("encoder.pkl", "rb") as f:32    encoder = pickle.load(f)33 34 35 36def inference_audio(audio):37    audio = whisper.load_audio(audio)38    audio = whisper.pad_or_trim(audio)39    40    mel = whisper.log_mel_spectrogram(audio).to(model.device)41    42    _, probs = model.detect_language(mel)43    44    options = whisper.DecodingOptions(fp16 = False)45    result = whisper.decode(model, mel, options)46    47    return result.text48 49def inference_text(audio):50    text =inference_audio(audio)51 52    sentiment_task = pipeline("sentiment-analysis", model=model_nlp, tokenizer=tokenizer)53    res=sentiment_task(text)[0]54 55    return text,res['label'],res['score']56 57    58def extract_features(data):59    # ZCR60    result = np.array([])61    zcr = np.mean(librosa.feature.zero_crossing_rate(y=data).T, axis=0)62    result=np.hstack((result, zcr)) # stacking horizontally63 64    # Chroma_stft65    stft = np.abs(librosa.stft(data))66    chroma_stft = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T, axis=0)67    result = np.hstack((result, chroma_stft)) # stacking horizontally68 69    # MFCC70    mfcc = np.mean(librosa.feature.mfcc(y=data, sr=sample_rate).T, axis=0)71    result = np.hstack((result, mfcc)) # stacking horizontally72 73    # Root Mean Square Value74    rms = np.mean(librosa.feature.rms(y=data).T, axis=0)75    result = np.hstack((result, rms)) # stacking horizontally76 77    # MelSpectogram78    mel = np.mean(librosa.feature.melspectrogram(y=data, sr=sample_rate).T, axis=0)79    result = np.hstack((result, mel)) # stacking horizontally80    81    return result82"""83def audio_emotions(audio):84    sr,data = audio85    features_audio = extract_features(data)86    features_audio = np.array(features_audio)87    scaled_features=scaler.transform(features_audio)88    scaled_features = np.expand_dims(scaled_features, axis=2)89    prediction=reloaded_model.predict(scaled_features)90    y_pred = encoder.inverse_transform(prediction)91    return y_pred92"""93def main(audio):94    r1,r2,r3=inference_text(audio)95    #r3=audio_emotions(audio)96    return r1,r2,r397    98 99audio = gr.Audio(100                    label="Input Audio",101                    show_label=False,102                    source="microphone",103                    type="filepath"104                )105 106 107app=gr.Interface(title="Sentiment Audio Analysis",fn=main,inputs=audio, outputs=["text","text","text"]).launch(debug = True)