justyoung/DiffSinger

1import librosa2import numpy as np3from pycwt import wavelet4from scipy.interpolate import interp1d5 6 7def load_wav(wav_file, sr):8    wav, _ = librosa.load(wav_file, sr=sr, mono=True)9    return wav10 11 12def convert_continuos_f0(f0):13    '''CONVERT F0 TO CONTINUOUS F014    Args:15        f0 (ndarray): original f0 sequence with the shape (T)16    Return:17        (ndarray): continuous f0 with the shape (T)18    '''19    # get uv information as binary20    f0 = np.copy(f0)21    uv = np.float32(f0 != 0)22 23    # get start and end of f024    if (f0 == 0).all():25        print("| all of the f0 values are 0.")26        return uv, f027    start_f0 = f0[f0 != 0][0]28    end_f0 = f0[f0 != 0][-1]29 30    # padding start and end of f0 sequence31    start_idx = np.where(f0 == start_f0)[0][0]32    end_idx = np.where(f0 == end_f0)[0][-1]33    f0[:start_idx] = start_f034    f0[end_idx:] = end_f035 36    # get non-zero frame index37    nz_frames = np.where(f0 != 0)[0]38 39    # perform linear interpolation40    f = interp1d(nz_frames, f0[nz_frames])41    cont_f0 = f(np.arange(0, f0.shape[0]))42 43    return uv, cont_f044 45 46def get_cont_lf0(f0, frame_period=5.0):47    uv, cont_f0_lpf = convert_continuos_f0(f0)48    # cont_f0_lpf = low_pass_filter(cont_f0_lpf, int(1.0 / (frame_period * 0.001)), cutoff=20)49    cont_lf0_lpf = np.log(cont_f0_lpf)50    return uv, cont_lf0_lpf51 52 53def get_lf0_cwt(lf0):54    '''55    input:56        signal of shape (N)57    output:58        Wavelet_lf0 of shape(10, N), scales of shape(10)59    '''60    mother = wavelet.MexicanHat()61    dt = 0.00562    dj = 163    s0 = dt * 264    J = 965 66    Wavelet_lf0, scales, _, _, _, _ = wavelet.cwt(np.squeeze(lf0), dt, dj, s0, J, mother)67    # Wavelet.shape => (J + 1, len(lf0))68    Wavelet_lf0 = np.real(Wavelet_lf0).T69    return Wavelet_lf0, scales70 71 72def norm_scale(Wavelet_lf0):73    Wavelet_lf0_norm = np.zeros((Wavelet_lf0.shape[0], Wavelet_lf0.shape[1]))74    mean = Wavelet_lf0.mean(0)[None, :]75    std = Wavelet_lf0.std(0)[None, :]76    Wavelet_lf0_norm = (Wavelet_lf0 - mean) / std77    return Wavelet_lf0_norm, mean, std78 79 80def normalize_cwt_lf0(f0, mean, std):81    uv, cont_lf0_lpf = get_cont_lf0(f0)82    cont_lf0_norm = (cont_lf0_lpf - mean) / std83    Wavelet_lf0, scales = get_lf0_cwt(cont_lf0_norm)84    Wavelet_lf0_norm, _, _ = norm_scale(Wavelet_lf0)85 86    return Wavelet_lf0_norm87 88 89def get_lf0_cwt_norm(f0s, mean, std):90    uvs = list()91    cont_lf0_lpfs = list()92    cont_lf0_lpf_norms = list()93    Wavelet_lf0s = list()94    Wavelet_lf0s_norm = list()95    scaless = list()96 97    means = list()98    stds = list()99    for f0 in f0s:100        uv, cont_lf0_lpf = get_cont_lf0(f0)101        cont_lf0_lpf_norm = (cont_lf0_lpf - mean) / std102 103        Wavelet_lf0, scales = get_lf0_cwt(cont_lf0_lpf_norm)  # [560,10]104        Wavelet_lf0_norm, mean_scale, std_scale = norm_scale(Wavelet_lf0)  # [560,10],[1,10],[1,10]105 106        Wavelet_lf0s_norm.append(Wavelet_lf0_norm)107        uvs.append(uv)108        cont_lf0_lpfs.append(cont_lf0_lpf)109        cont_lf0_lpf_norms.append(cont_lf0_lpf_norm)110        Wavelet_lf0s.append(Wavelet_lf0)111        scaless.append(scales)112        means.append(mean_scale)113        stds.append(std_scale)114 115    return Wavelet_lf0s_norm, scaless, means, stds116 117 118def inverse_cwt_torch(Wavelet_lf0, scales):119    import torch120    b = ((torch.arange(0, len(scales)).float().to(Wavelet_lf0.device)[None, None, :] + 1 + 2.5) ** (-2.5))121    lf0_rec = Wavelet_lf0 * b122    lf0_rec_sum = lf0_rec.sum(-1)123    lf0_rec_sum = (lf0_rec_sum - lf0_rec_sum.mean(-1, keepdim=True)) / lf0_rec_sum.std(-1, keepdim=True)124    return lf0_rec_sum125 126 127def inverse_cwt(Wavelet_lf0, scales):128    b = ((np.arange(0, len(scales))[None, None, :] + 1 + 2.5) ** (-2.5))129    lf0_rec = Wavelet_lf0 * b130    lf0_rec_sum = lf0_rec.sum(-1)131    lf0_rec_sum = (lf0_rec_sum - lf0_rec_sum.mean(-1, keepdims=True)) / lf0_rec_sum.std(-1, keepdims=True)132    return lf0_rec_sum133 134 135def cwt2f0(cwt_spec, mean, std, cwt_scales):136    assert len(mean.shape) == 1 and len(std.shape) == 1 and len(cwt_spec.shape) == 3137    import torch138    if isinstance(cwt_spec, torch.Tensor):139        f0 = inverse_cwt_torch(cwt_spec, cwt_scales)140        f0 = f0 * std[:, None] + mean[:, None]141        f0 = f0.exp()  # [B, T]142    else:143        f0 = inverse_cwt(cwt_spec, cwt_scales)144        f0 = f0 * std[:, None] + mean[:, None]145        f0 = np.exp(f0)  # [B, T]146    return f0147