echodict/llama.cpp

1#pragma once2 3#include "ggml.h"4#include "clip-model.h"5 6#include <cstdint>7#include <vector>8#include <string>9 10#define MTMD_INTERNAL_HEADER11 12struct mtmd_audio_mel {13    int n_len;14    int n_len_org;15    int n_mel;16 17    std::vector<float> data;18};19 20struct mtmd_audio_mel_filters {21    int32_t n_mel;22    int32_t n_fft;23 24    std::vector<float> data;25};26 27// cache for audio processing, each processor instance owns its own cache28struct mtmd_audio_cache {29    std::vector<float> sin_vals;30    std::vector<float> cos_vals;31 32    std::vector<float> hann_window;33 34    mtmd_audio_mel_filters filters;35 36    void fill_sin_cos_table(uint32_t n);37 38    void fill_hann_window(uint32_t length, bool periodic);39 40    // Build mel filterbank matrix [n_mel × n_fft_bins] at runtime.41    // n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.42    void fill_mel_filterbank_matrix(int   n_mel,43                                    int   n_fft,44                                    int   sample_rate,               // e.g. 1600045                                    float fmin             = 0.0f,   // e.g. 0.046                                    float fmax             = -1.0f,  // e.g. sr/2; pass -1 for auto47                                    bool  slaney_area_norm = true,48                                    float scale            = 1.0f,49                                    bool  use_htk          = false50    );51};52 53struct mtmd_audio_preprocessor {54    const clip_hparams & hparams;55 56    mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}57 58    virtual ~mtmd_audio_preprocessor() = default;59    virtual void initialize() = 0; // NOT thread-safe60    virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) = 0;61};62 63struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {64    mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}65    void initialize() override;66    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;67 68  private:69    mtmd_audio_cache cache;70};71 72struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {73    mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}74    void initialize() override;75    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;76 77  private:78    mtmd_audio_cache cache;79};80 81struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {82    mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}83    void initialize() override;84    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;85 86  private:87    mtmd_audio_cache cache;88};89 90//91// streaming ISTFT - converts spectrogram frames back to audio one frame at a time92//93struct mtmd_audio_streaming_istft {94    mtmd_audio_streaming_istft(int n_fft, int hop_length);95 96    // reset streaming state97    void reset();98 99    // process a single STFT frame (streaming)100    // frame_spectrum: [n_fft_bins x 2] interleaved real/imag101    // returns: up to hop_length samples102    std::vector<float> process_frame(const float * frame_spectrum);103 104    // flush remaining samples at end of stream105    std::vector<float> flush();106 107  private:108    int n_fft;109    int hop_length;110    int n_fft_bins;111 112    // Own cache for output processing113    mtmd_audio_cache cache;114 115    // Streaming state116    std::vector<float> overlap_buffer;117    std::vector<float> window_sum_buffer;118    int                padding_to_remove;119 120    // Working buffers for IFFT121    std::vector<float> ifft_in;122    std::vector<float> ifft_out;123};124