echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0773
1#include "mtmd-audio.h"2 3#define _USE_MATH_DEFINES // for M_PI4#include <cmath>5#include <cstdint>6#include <cstring>7#include <thread>8#include <vector>9#include <fstream>10#include <algorithm>11#include <functional>12 13// some of the code here is copied from whisper.cpp14 15constexpr bool DEBUG = false;16 17void mtmd_audio_cache::fill_sin_cos_table(uint32_t n) {18 sin_vals.resize(n);19 cos_vals.resize(n);20 for (uint32_t i = 0; i < n; i++) {21 double theta = (2 * M_PI * i) / n;22 sin_vals[i] = sinf(theta);23 cos_vals[i] = cosf(theta);24 }25}26 27void mtmd_audio_cache::fill_hann_window(uint32_t length, bool periodic) {28 hann_window.resize(length);29 int offset = periodic ? 0 : -1;30 for (uint32_t i = 0; i < length; i++) {31 hann_window[i] = 0.5 * (1.0 - cosf((2.0 * M_PI * i) / (length + offset)));32 }33}34 35void mtmd_audio_cache::fill_mel_filterbank_matrix(int n_mel,36 int n_fft,37 int sample_rate,38 float fmin,39 float fmax,40 bool slaney_area_norm,41 float scale,42 bool use_htk) {43 GGML_ASSERT(n_mel > 0 && n_fft > 1);44 if (fmax <= 0.0f) {45 fmax = 0.5f * sample_rate;46 }47 48 std::function<double(double)> hz_to_mel;49 std::function<double(double)> mel_to_hz;50 51 if (use_htk) {52 hz_to_mel = [](const double f_hz) -> double {53 return 2595.0 * log10(1.0 + f_hz / 700.0);54 };55 mel_to_hz = [](const double m) -> double {56 return 700.0 * (pow(10.0, m / 2595.0) - 1.0);57 };58 } else {59 // Slaney scale (matches librosa default)60 const double min_log_hz = 1000.0;61 const double lin_slope = 3 / 200.;62 const double min_log_mel = min_log_hz * lin_slope;63 const double log_step = log(6.4) / 27.0;64 hz_to_mel = [min_log_hz, lin_slope, log_step, min_log_mel](const double f_hz) -> double {65 return (f_hz < min_log_hz) ? f_hz * lin_slope : min_log_mel + log(f_hz / min_log_hz) / log_step;66 };67 mel_to_hz = [min_log_hz, lin_slope, log_step, min_log_mel](const double m) -> double {68 return (m < min_log_mel) ? m / lin_slope : min_log_hz * exp((m - min_log_mel) * log_step);69 };70 }71 72 // infer N_fft from n_fft_bins73 const double bin_hz_step = double(sample_rate) / double(n_fft);74 75 // mel grid: n_mel + 2 edges76 const double m_lo = hz_to_mel(fmin);77 const double m_hi = hz_to_mel(fmax);78 std::vector<double> mel_pts(n_mel + 2);79 for (int i = 0; i < n_mel + 2; ++i) {80 mel_pts[i] = m_lo + (m_hi - m_lo) * (double(i) / (n_mel + 1));81 }82 83 // convert to Hz84 std::vector<double> hz_pts(n_mel + 2);85 for (int i = 0; i < n_mel + 2; ++i) {86 hz_pts[i] = mel_to_hz(mel_pts[i]);87 }88 89 const int n_fft_bins = n_fft / 2 + 1;90 91 // filterbank92 std::vector<float> out(n_mel * n_fft_bins, 0);93 for (int m = 0; m < n_mel; ++m) {94 const double f_left = hz_pts[m];95 const double f_center = hz_pts[m + 1];96 const double f_right = hz_pts[m + 2];97 98 const double denom_l = std::max(1e-30, f_center - f_left);99 const double denom_r = std::max(1e-30, f_right - f_center);100 const double enorm = slaney_area_norm ? (2.0 / std::max(1e-30, f_right - f_left)) : 1.0;101 102 for (int k = 0; k < n_fft_bins; ++k) {103 const double f = k * bin_hz_step;104 double w = 0.0;105 if (f >= f_left && f <= f_center) {106 w = (f - f_left) / denom_l;107 } else if (f > f_center && f <= f_right) {108 w = (f_right - f) / denom_r;109 }110 out[size_t(m) * size_t(n_fft_bins) + size_t(k)] = float(w * enorm * scale);111 }112 }113 114 filters.n_mel = n_mel;115 filters.n_fft = n_fft;116 filters.data = std::move(out);117 118 if (DEBUG) { // debug119 for (size_t i = 0; i < filters.data.size(); ++i) {120 if (filters.data[i] != 0.0f) {121 printf("filters[%zu] = %f\n", i, filters.data[i] * 1000.0f);122 }123 }124 }125}126 127// Unified DFT implementation for both forward and inverse transforms128// Template parameters:129// Inverse: false = DFT with exp(-2πi·k·n/N), no scaling130// true = IDFT with exp(+2πi·k·n/N), scales by 1/N131// RealInput: true = input is real-valued (stride 1), avoids imaginary computations132// false = input is complex-valued (interleaved real/imag, stride 2)133template <bool Inverse, bool RealInput>134static void dft_impl(const mtmd_audio_cache & cache, const float * in, int N, float * out) {135 const int n_sin_cos_vals = cache.sin_vals.size();136 const int sin_cos_step = n_sin_cos_vals / N;137 138 constexpr float sign = Inverse ? 1.0f : -1.0f;139 const float scale = Inverse ? (1.0f / N) : 1.0f;140 141 for (int k = 0; k < N; k++) {142 float re = 0;143 float im = 0;144 145 for (int n = 0; n < N; n++) {146 int idx = (k * n * sin_cos_step) % n_sin_cos_vals;147 float cos_val = cache.cos_vals[idx];148 float sin_val = cache.sin_vals[idx];149 150 if constexpr (RealInput) {151 // Real input: in_im = 0, simplifies to:152 // re += in_re * cos_val153 // im += sign * in_re * sin_val154 float in_re = in[n];155 re += in_re * cos_val;156 im += sign * in_re * sin_val;157 } else {158 float in_re = in[n * 2 + 0];159 float in_im = in[n * 2 + 1];160 // (a + bi) * (cos + sign*i*sin) = (a*cos - sign*b*sin) + (sign*a*sin + b*cos)i161 re += in_re * cos_val - sign * in_im * sin_val;162 im += sign * in_re * sin_val + in_im * cos_val;163 }164 }165 166 out[k * 2 + 0] = re * scale;167 out[k * 2 + 1] = im * scale;168 }169}170 171// Cooley-Tukey FFT/IFFT unified implementation172// Template parameters:173// Inverse: false = FFT with exp(-2πi·k/N), no scaling174// true = IFFT with exp(+2πi·k/N), scales by 0.5 at each level175// RealInput: true = input is real-valued (stride 1)176// false = input is complex-valued (interleaved real/imag, stride 2)177template <bool Inverse, bool RealInput>178static void fft_impl(const mtmd_audio_cache & cache, float * in, int N, float * out) {179 GGML_ASSERT(N > 0);180 const int n_sin_cos_vals = cache.sin_vals.size();181 182 if (N == 1) {183 out[0] = in[0];184 if constexpr (RealInput) {185 out[1] = 0.0f;186 } else {187 out[1] = in[1];188 }189 return;190 }191 192 const int half_N = N / 2;193 if (N - half_N * 2 == 1) {194 // Odd N: fall back to DFT195 dft_impl<Inverse, RealInput>(cache, in, N, out);196 return;197 }198 199 // Split into even and odd200 if constexpr (RealInput) {201 // Real input: stride is 1, copy only real values202 float * even = in + N;203 for (int i = 0; i < half_N; ++i) {204 even[i] = in[2 * i];205 }206 float * even_fft = out + 2 * N;207 fft_impl<Inverse, true>(cache, even, half_N, even_fft);208 209 float * odd = even;210 for (int i = 0; i < half_N; ++i) {211 odd[i] = in[2 * i + 1];212 }213 float * odd_fft = even_fft + N;214 fft_impl<Inverse, true>(cache, odd, half_N, odd_fft);215 } else {216 // Complex input: stride is 2, copy complex pairs217 float * even = in + N * 2;218 for (int i = 0; i < half_N; ++i) {219 even[i * 2 + 0] = in[2 * i * 2 + 0];220 even[i * 2 + 1] = in[2 * i * 2 + 1];221 }222 float * even_fft = out + 2 * N;223 fft_impl<Inverse, false>(cache, even, half_N, even_fft);224 225 float * odd = even;226 for (int i = 0; i < half_N; ++i) {227 odd[i * 2 + 0] = in[(2 * i + 1) * 2 + 0];228 odd[i * 2 + 1] = in[(2 * i + 1) * 2 + 1];229 }230 float * odd_fft = even_fft + N;231 fft_impl<Inverse, false>(cache, odd, half_N, odd_fft);232 }233 234 float * even_fft = out + 2 * N;235 float * odd_fft = even_fft + N;236 237 const int sin_cos_step = n_sin_cos_vals / N;238 239 constexpr float sign = Inverse ? 1.0f : -1.0f;240 constexpr float scale = Inverse ? 0.5f : 1.0f;241 242 for (int k = 0; k < half_N; k++) {243 int idx = k * sin_cos_step; // t = 2*M_PI*k/N244 float re = cache.cos_vals[idx];245 float im = sign * cache.sin_vals[idx];246 247 float re_odd = odd_fft[2 * k + 0];248 float im_odd = odd_fft[2 * k + 1];249 250 out[2 * k + 0] = scale * (even_fft[2 * k + 0] + re * re_odd - im * im_odd);251 out[2 * k + 1] = scale * (even_fft[2 * k + 1] + re * im_odd + im * re_odd);252 253 out[2 * (k + half_N) + 0] = scale * (even_fft[2 * k + 0] - re * re_odd + im * im_odd);254 out[2 * (k + half_N) + 1] = scale * (even_fft[2 * k + 1] - re * im_odd - im * re_odd);255 }256}257 258// Forward FFT for real input (used by mel spectrogram)259static void fft(const mtmd_audio_cache & cache, float * in, int N, float * out) {260 fft_impl<false, true>(cache, in, N, out);261}262 263// Inverse FFT for complex input264static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out) {265 fft_impl<true, false>(cache, in, N, out);266}267 268struct filter_params {269 int32_t n_mel;270 int32_t n_fft_bins;271 int32_t hann_window_size;272 int32_t hop_length;273 int32_t sample_rate;274 bool no_padding = false;275 bool center_padding = false;276 float preemph = 0.f;277 bool use_natural_log = false;278 bool norm_per_feature = false;279 bool use_magnitude = false; // |X| instead of |X|^2280 float mel_floor = 5.960464477539063e-08f;281};282 283static void log_mel_spectrogram_worker_thread(int ith,284 const float * hann,285 const std::vector<float> & samples,286 int n_samples,287 int frame_size,288 int frame_step,289 int n_threads,290 const filter_params & params,291 const mtmd_audio_cache & cache,292 mtmd_audio_mel & out) {293 std::vector<float> fft_in(frame_size * 2, 0.0);294 std::vector<float> fft_out(frame_size * 2 * 2 * 2);295 296 int n_fft_bins = params.n_fft_bins;297 int i = ith;298 299 const auto & filters = cache.filters;300 301 // make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist302 GGML_ASSERT(n_fft_bins == 1 + (frame_size / 2));303 GGML_ASSERT(cache.sin_vals.size() == cache.cos_vals.size());304 // calculate FFT only when fft_in are not all zero305 for (; i < std::min(n_samples / frame_step + 1, out.n_len); i += n_threads) {306 const int offset = i * frame_step;307 308 // apply Hann window (~10% faster)309 for (int j = 0; j < std::min(frame_size, n_samples - offset); j++) {310 fft_in[j] = hann[j] * samples[offset + j];311 }312 313 // fill the rest with zeros314 if (n_samples - offset < frame_size) {315 std::fill(fft_in.begin() + (n_samples - offset), fft_in.end(), 0.0);316 }317 318 // FFT319 fft(cache, fft_in.data(), frame_size, fft_out.data());320 321 // Calculate modulus^2 (power) or modulus (magnitude)322 for (int j = 0; j < n_fft_bins; j++) {323 float power = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);324 fft_out[j] = params.use_magnitude ? sqrtf(power) : power;325 }326 327 // mel spectrogram328 for (int j = 0; j < out.n_mel; j++) {329 double sum = 0.0;330 // unroll loop (suggested by GH user @lunixbochs)331 int k = 0;332 for (k = 0; k < n_fft_bins - 3; k += 4) {333 size_t idx = size_t(j) * size_t(n_fft_bins) + size_t(k);334 sum +=335 fft_out[k + 0] * filters.data[idx + 0] +336 fft_out[k + 1] * filters.data[idx + 1] +337 fft_out[k + 2] * filters.data[idx + 2] +338 fft_out[k + 3] * filters.data[idx + 3];339 }340 // handle n_fft remainder341 for (; k < n_fft_bins; k++) {342 sum += fft_out[k] * filters.data[j * n_fft_bins + k];343 }344 sum = std::max(sum, (double)params.mel_floor);345 sum = params.use_natural_log346 ? log(sum)347 : log10(sum);348 out.data[j * out.n_len + i] = sum;349 }350 }351 352 // Otherwise fft_out are all zero353 double sum = params.use_natural_log ? log(1e-10) : log10(1e-10);354 for (; i < out.n_len; i += n_threads) {355 for (int j = 0; j < out.n_mel; j++) {356 out.data[j * out.n_len + i] = sum;357 }358 }359}360 361// ref: https://github.com/openai/whisper/blob/main/whisper/audio.py#L110-L157362static bool log_mel_spectrogram(363 const float * samples,364 const int n_samples_in,365 const int n_threads,366 const filter_params & params,367 const mtmd_audio_cache & cache,368 mtmd_audio_mel & out) {369 //const int64_t t_start_us = ggml_time_us();370 371 out.n_len_org = n_samples_in;372 int n_samples = n_samples_in;373 374 // Hann window375 const float * hann = cache.hann_window.data();376 const int frame_size = (params.n_fft_bins - 1) * 2;377 const int frame_step = params.hop_length;378 379 // Padding380 std::vector<float> samples_padded;381 if (params.no_padding) {382 // no padding, use samples as-is383 samples_padded = std::vector<float>(samples, samples + n_samples);384 samples = samples_padded.data();385 n_samples = samples_padded.size();386 } else if (params.center_padding) {387 const auto pad_amount = frame_size / 2;388 samples_padded = std::vector<float>(n_samples + 2 * pad_amount, 0);389 std::copy(samples, samples + n_samples, samples_padded.data() + pad_amount);390 samples = samples_padded.data();391 n_samples = samples_padded.size();392 } else {393 // existing padding logic394 int64_t stage_1_pad = params.sample_rate * 30;395 int64_t stage_2_pad = frame_size / 2;396 samples_padded.resize(n_samples + stage_1_pad + stage_2_pad * 2);397 std::copy(samples, samples + n_samples, samples_padded.begin() + stage_2_pad);398 // pad 30 seconds of zeros at the end of audio (480,000 samples) + reflective pad 200 samples at the end of audio399 std::fill(samples_padded.begin() + n_samples + stage_2_pad, samples_padded.begin() + n_samples + stage_1_pad + 2 * stage_2_pad, 0);400 // reflective pad 200 samples at the beginning of audio401 if (n_samples < stage_2_pad + 1) {402 // TODO: Handle short audio differently or return error403 return false;404 }405 std::reverse_copy(samples + 1, samples + 1 + stage_2_pad, samples_padded.begin());406 }407 408 // preemphasis409 if (params.preemph) {410 const int pad_amount = frame_size / 2;411 const float preemph = 0.97f;412 float prev = samples_padded[pad_amount];413 for (int i = pad_amount + 1; i + pad_amount < n_samples; ++i) {414 float cur = samples_padded[i];415 samples_padded[i] = cur - preemph * prev;416 prev = cur;417 }418 }419 420 // pad hann window if it's smaller than frame_size421 // TODO: probably unnecessary here? (or better doing it in g_cache?)422 std::vector<float> hann_window_padded;423 if (params.hann_window_size < frame_size) {424 hann_window_padded.resize(frame_size);425 const int padding = (frame_size - params.hann_window_size) / 2;426 std::copy(hann, hann + params.hann_window_size, &hann_window_padded[padding]);427 hann = hann_window_padded.data();428 }429 430 431 GGML_ASSERT(params.n_fft_bins > 0);432 GGML_ASSERT(params.hop_length > 0);433 out.n_mel = params.n_mel;434 out.n_len = (n_samples - frame_size) / frame_step + 1;435 // TODO: handle these checks better436 if (out.n_mel > 0 && (unsigned long)out.n_len > SIZE_MAX / out.n_mel) {437 LOG_ERR("%s: size overflow\n", __func__);438 return false;439 }440 if (n_samples < frame_size) {441 LOG_ERR("%s: not enough samples after padding\n", __func__);442 return false;443 }444 out.data.resize(out.n_mel * out.n_len);445 446 {447 std::vector<std::thread> workers(n_threads - 1);448 for (int iw = 0; iw < n_threads - 1; ++iw) {449 workers[iw] =450 std::thread(log_mel_spectrogram_worker_thread, iw + 1, hann, std::cref(samples_padded), n_samples,451 frame_size, frame_step, n_threads, std::cref(params), std::cref(cache), std::ref(out));452 }453 454 // main thread455 log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples, frame_size, frame_step, n_threads, params,456 cache, out);457 for (int iw = 0; iw < n_threads - 1; ++iw) {458 workers[iw].join();459 }460 }461 462 const int effective_n_len = n_samples_in / frame_step;463 if (params.norm_per_feature) {464 GGML_ASSERT(effective_n_len > 1);465 for (int i = 0; i < out.n_mel; i++) {466 double mean = 0;467 for (int j = 0; j < effective_n_len; ++j) {468 mean += out.data[i * out.n_len + j];469 }470 mean /= effective_n_len;471 472 double var = 0.0;473 for (int j = 0; j < effective_n_len; ++j) {474 const double value = out.data[i * out.n_len + j] - mean;475 var += value * value;476 }477 var /= effective_n_len - 1; // unbiased478 const double mstd = std::sqrt(var + 1e-5);479 480 for (int j = 0; j < effective_n_len; ++j) {481 auto &value = out.data[i * out.n_len + j];482 value = (value - mean) / mstd;483 }484 485 // pad the rest with zeros486 for (int j = effective_n_len; j < out.n_len; ++j) {487 out.data[i * out.n_len + j] = 0.0;488 }489 }490 } else if (!params.no_padding) {491 // Whisper-style clamping and normalization (NOT used by Gemma4)492 double mmax = -1e20;493 for (int i = 0; i < out.n_mel*out.n_len; i++) {494 if (out.data[i] > mmax) {495 mmax = out.data[i];496 }497 }498 499 mmax -= 8.0;500 501 for (int i = 0; i < out.n_mel*out.n_len; i++) {502 if (out.data[i] < mmax) {503 out.data[i] = mmax;504 }505 out.data[i] = (out.data[i] + 4.0)/4.0;506 }507 }508 509 // Dump log_mel_spectrogram510 if (DEBUG) {511 std::ofstream outFile("log_mel_spectrogram.json");512 outFile << "[";513 for (uint64_t i = 0; i < out.data.size() - 1; i++) {514 outFile << out.data[i] << ", ";515 }516 outFile << out.data[out.data.size() - 1] << "]";517 outFile.close();518 }519 520 return true;521}522 523//524// mtmd_audio_preprocessor_whisper525//526 527void mtmd_audio_preprocessor_whisper::initialize() {528 cache.fill_sin_cos_table(hparams.audio_n_fft);529 cache.fill_hann_window(hparams.audio_window_len, true);530 cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);531}532 533bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples,534 size_t n_samples,535 std::vector<mtmd_audio_mel> & output) {536 if (n_samples == 0) {537 // empty audio538 return false;539 }540 541 std::vector<float> smpl;542 // if input is too short, pad with zeros543 // this is to avoid potential issues with stage1/2 padding in log_mel_spectrogram544 // TODO: maybe handle this better545 size_t min_samples = (size_t) hparams.audio_sample_rate * (hparams.audio_chunk_len + 1); // +1 second margin546 if (n_samples < min_samples) {547 smpl.resize(min_samples, 0.0f);548 std::memcpy(smpl.data(), samples, n_samples * sizeof(float));549 samples = smpl.data();550 n_samples = smpl.size();551 }552 553 filter_params params;554 params.n_mel = hparams.n_mel_bins;555 params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);556 params.hann_window_size = hparams.audio_window_len;557 params.hop_length = hparams.audio_hop_len;558 params.sample_rate = hparams.audio_sample_rate;559 params.center_padding = false;560 params.preemph = 0.0f; // disabled561 params.use_natural_log = false;562 params.norm_per_feature = false;563 564 // make sure the cache is initialized565 GGML_ASSERT(!cache.sin_vals.empty());566 GGML_ASSERT(!cache.cos_vals.empty());567 GGML_ASSERT(!cache.filters.data.empty());568 569 mtmd_audio_mel out_full;570 bool ok = log_mel_spectrogram(samples, n_samples,571 4, // n_threads572 params, cache, out_full);573 if (!ok) {574 return false;575 }576 577 // because the cgraph in clip.cpp only accepts 3000 frames each, we need to split the mel578 // we always expect the mel to have 3000 silent frames at the end579 if (DEBUG) {580 printf("output: n_mel = %d, n_len = %d\n", out_full.n_mel, out_full.n_len);581 }582 const size_t frames_per_chunk = 3000;583 GGML_ASSERT((size_t) out_full.n_len > frames_per_chunk);584 for (size_t off = 0; off < (size_t) out_full.n_len; off += frames_per_chunk) {585 int n_len = std::min(frames_per_chunk, (size_t) out_full.n_len - off);586 if ((size_t) n_len < frames_per_chunk) {587 break; // last incomplete chunk will always be a padded chunk, safe to ignore588 }589 590 mtmd_audio_mel out_chunk;591 out_chunk.n_len = n_len;592 out_chunk.n_mel = out_full.n_mel;593 out_chunk.n_len_org = out_full.n_mel; // unused594 out_chunk.data.reserve(out_chunk.n_mel * out_chunk.n_len);595 596 for (int i = 0; i < out_full.n_mel; i++) {597 auto src = out_full.data.begin() + i * out_full.n_len + off;598 out_chunk.data.insert(out_chunk.data.end(), src, src + frames_per_chunk);599 }600 601 output.push_back(std::move(out_chunk));602 }603 604 return true;605}606 607//608// mtmd_audio_preprocessor_conformer609//610 611void mtmd_audio_preprocessor_conformer::initialize() {612 cache.fill_sin_cos_table(hparams.audio_n_fft);613 cache.fill_hann_window(hparams.audio_window_len, true);614 cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);615}616 617bool mtmd_audio_preprocessor_conformer::preprocess(const float * samples,618 size_t n_samples,619 std::vector<mtmd_audio_mel> & output) {620 // empty audio621 if (n_samples == 0) {622 return false;623 }624 625 filter_params params;626 params.n_mel = hparams.n_mel_bins;627 params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);628 params.hann_window_size = hparams.audio_window_len;629 params.hop_length = hparams.audio_hop_len;630 params.sample_rate = hparams.audio_sample_rate;631 params.center_padding = true;632 params.preemph = 0.97f;633 params.use_natural_log = true;634 params.norm_per_feature = true;635 636 // make sure the cache is initialized637 GGML_ASSERT(!cache.sin_vals.empty());638 GGML_ASSERT(!cache.cos_vals.empty());639 GGML_ASSERT(!cache.filters.data.empty());640 641 mtmd_audio_mel out_full;642 bool ok = log_mel_spectrogram(samples, n_samples,643 4, // n_threads644 params, cache, out_full);645 if (!ok) {646 return false;647 }648 649 output.push_back(std::move(out_full));650 return true;651}652 653//654// mtmd_audio_preprocessor_gemma4a655//656 657void mtmd_audio_preprocessor_gemma4a::initialize() {658 cache.fill_sin_cos_table(hparams.audio_n_fft);659 660 // Standard periodic Hann window, zero-padded to FFT size661 cache.hann_window.assign(hparams.audio_n_fft, 0.0f);662 for (uint32_t i = 0; i < (uint32_t)hparams.audio_window_len; i++) {663 cache.hann_window[i] = 0.5f - 0.5f * cosf((2.0f * (float)M_PI * i) / hparams.audio_window_len);664 }665 666 // HTK mel scale, no Slaney area normalization667 cache.fill_mel_filterbank_matrix(668 hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate,669 0.0f, hparams.audio_sample_rate / 2.0f,670 /*slaney_area_norm=*/ false,671 /*scale=*/ 1.0f,672 /*use_htk=*/ true673 );674}675 676bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * samples,677 size_t n_samples,678 std::vector<mtmd_audio_mel> & output) {679 if (n_samples == 0) {680 return false;681 }682 683 GGML_ASSERT(!cache.sin_vals.empty());684 GGML_ASSERT(!cache.cos_vals.empty());685 GGML_ASSERT(!cache.filters.data.empty());686 687 filter_params params;688 params.n_mel = hparams.n_mel_bins;689 params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);690 params.hann_window_size = hparams.audio_n_fft; // window is zero-padded to FFT size691 params.hop_length = hparams.audio_hop_len;692 params.sample_rate = hparams.audio_sample_rate;693 params.no_padding = true;694 params.center_padding = false;695 params.preemph = 0.0f;696 params.use_natural_log = true;697 params.use_magnitude = true;698 params.mel_floor = 0.001f;699 params.norm_per_feature = false;700 701 // Split into 30-second chunks (model context limit, ~750 tokens each)702 const size_t chunk_samples = 30 * hparams.audio_sample_rate;703 for (size_t off = 0; off < n_samples; off += chunk_samples) {704 const float * chunk_ptr = samples + off;705 size_t chunk_len = std::min(chunk_samples, n_samples - off);706 707 // Semicausal left-padding + right-padding to match PyTorch frame count708 const int pad_left = hparams.audio_window_len / 2;709 const int fft_size = hparams.audio_n_fft;710 const int hop = hparams.audio_hop_len;711 const int n_with_left = (int)chunk_len + pad_left;712 // PyTorch: unfold(size=frame_length+1, step=hop) on semicausal-padded waveform713 const int pt_frames = (n_with_left - (hparams.audio_window_len + 1)) / hop + 1;714 const int n_padded_needed = (pt_frames - 1) * hop + fft_size;715 const int total_pad = std::max((int)(n_padded_needed - (int)chunk_len), pad_left);716 std::vector<float> padded_samples(total_pad + chunk_len, 0.0f);717 std::copy(chunk_ptr, chunk_ptr + chunk_len, padded_samples.data() + pad_left);718 719 mtmd_audio_mel out_chunk;720 bool ok = log_mel_spectrogram(padded_samples.data(), padded_samples.size(), 4, params, cache, out_chunk);721 if (!ok) {722 return false;723 }724 725 // Trim to PyTorch frame count726 out_chunk.n_len = std::min(out_chunk.n_len, pt_frames);727 728 output.push_back(std::move(out_chunk));729 }730 731 return true;732}733 734//735// mtmd_audio_streaming_istft implementation736//737 738mtmd_audio_streaming_istft::mtmd_audio_streaming_istft(int n_fft, int hop_length) :739 n_fft(n_fft),740 hop_length(hop_length),741 n_fft_bins(n_fft / 2 + 1),742 overlap_buffer(n_fft, 0.0f),743 window_sum_buffer(n_fft, 0.0f),744 padding_to_remove((n_fft - hop_length) / 2),745 ifft_in(n_fft * 2 * 4, 0.0f), // extra space for recursive IFFT746 ifft_out(n_fft * 2 * 4, 0.0f) {747 GGML_ASSERT(n_fft > 0 && hop_length > 0 && hop_length <= n_fft);748 cache.fill_sin_cos_table(n_fft);749 cache.fill_hann_window(n_fft, true);750}751 752void mtmd_audio_streaming_istft::reset() {753 std::fill(overlap_buffer.begin(), overlap_buffer.end(), 0.0f);754 std::fill(window_sum_buffer.begin(), window_sum_buffer.end(), 0.0f);755 padding_to_remove = (n_fft - hop_length) / 2;756}757 758std::vector<float> mtmd_audio_streaming_istft::process_frame(const float * frame_spectrum) {759 std::vector<float> output(hop_length);760 761 // copy frequencies762 for (int j = 0; j < n_fft_bins; j++) {763 ifft_in[j * 2 + 0] = frame_spectrum[j * 2 + 0];764 ifft_in[j * 2 + 1] = frame_spectrum[j * 2 + 1];765 }766 767 // mirror negative frequencies768 for (int j = 1; j < n_fft_bins - 1; j++) {769 int mirror_idx = n_fft - j;770 ifft_in[mirror_idx * 2 + 0] = ifft_in[j * 2 + 0];771 ifft_in[mirror_idx * 2 + 1] = -ifft_in[j * 2 + 1]; // conjugate772 }773 774 ifft(cache, ifft_in.data(), n_fft, ifft_out.data());775 776 // update window sum and overlap buffer777 for (int j = 0; j < n_fft; j++) {778 window_sum_buffer[j] += cache.hann_window[j] * cache.hann_window[j];779 overlap_buffer[j] += ifft_out[j * 2] * cache.hann_window[j];780 }781 782 // extract hop_length samples with normalization783 for (int i = 0; i < hop_length; i++) {784 if (window_sum_buffer[i] > 1e-8f) {785 output[i] = overlap_buffer[i] / window_sum_buffer[i];786 } else {787 output[i] = overlap_buffer[i];788 }789 }790 791 // shift buffers left by hop_length792 std::copy(overlap_buffer.begin() + hop_length, overlap_buffer.end(), overlap_buffer.begin());793 std::fill(overlap_buffer.end() - hop_length, overlap_buffer.end(), 0.0f);794 795 std::copy(window_sum_buffer.begin() + hop_length, window_sum_buffer.end(), window_sum_buffer.begin());796 std::fill(window_sum_buffer.end() - hop_length, window_sum_buffer.end(), 0.0f);797 798 // Remove padding if needed799 int to_remove = std::min(padding_to_remove, (int) output.size());800 padding_to_remove -= to_remove;801 output.erase(output.begin(), output.begin() + to_remove);802 803 return output;804}805 806std::vector<float> mtmd_audio_streaming_istft::flush() {807 std::vector<float> output;808 809 // Extract remaining samples from overlap buffer810 // Continue until we've extracted all meaningful samples811 int remaining = n_fft - hop_length;812 while (remaining > 0) {813 int chunk_size = std::min(remaining, hop_length);814 815 for (int i = 0; i < chunk_size; i++) {816 float sample;817 if (window_sum_buffer[i] > 1e-8f) {818 sample = overlap_buffer[i] / window_sum_buffer[i];819 } else {820 sample = overlap_buffer[i];821 }822 output.push_back(sample);823 }824 825 // Shift buffers826 std::copy(overlap_buffer.begin() + chunk_size, overlap_buffer.end(), overlap_buffer.begin());827 std::fill(overlap_buffer.end() - chunk_size, overlap_buffer.end(), 0.0f);828 829 std::copy(window_sum_buffer.begin() + chunk_size, window_sum_buffer.end(), window_sum_buffer.begin());830 std::fill(window_sum_buffer.end() - chunk_size, window_sum_buffer.end(), 0.0f);831 832 remaining -= chunk_size;833 }834 835 return output;836}837