AudioEmbedding.cpp

Go to the documentation of this file.

#include <fstream>
 
namespace krotos
{
 
//=====================================================================================
AudioEmbedding::AudioEmbedding()
    : m_fftSize(2048), m_hopSize(1024), m_fft(static_cast<int>(std::log2(m_fftSize))),
      m_window(m_fftSize + 1, dsp::WindowingFunction<float>::hann, false)
{
    initialise();
}
 
void AudioEmbedding::initialise()
{
    // Note: if these params change the autoencoder should be retrained
    const int sr = 48000;
    const int n_mels = 40;
    const float fmin = 20.f;
    const float fmax = 8000.f;
    const bool htk = false;
    m_mel = AudioEmbedding::mel(sr, m_fftSize, n_mels, fmin, fmax, htk);
    m_initialised = true;
}
 
std::vector<float> AudioEmbedding::forward(const AudioSampleBuffer& buffer)
{
    assert(m_initialised);
 
    const auto inputSize = buffer.getNumSamples();
    const auto inputData = buffer.getReadPointer(0);
 
    std::vector<float> sigFrame(2 * m_fftSize);
    std::vector<float> embedding(m_mel.size(), 0.f);
 
    std::size_t numFrames = 0;
    for (int posin = 0; posin < inputSize; posin += m_hopSize)
    {
        // extract a signal frame
        for (int i = 0; i < m_fftSize; ++i)
        {
            if (posin + i < inputSize)
                sigFrame.at(i) = inputData[posin + i];
            else
                sigFrame.at(i) = 0.f;
        }
 
        // apply window function signal frame
        m_window.multiplyWithWindowingTable(sigFrame.data(), m_fftSize);
 
        // apply FFT on the frame - get back frequency magnitude
        m_fft.performFrequencyOnlyForwardTransform(sigFrame.data(), false);
 
        // apply mel filterbank
        auto melFrame = applyMelFilterbank(sigFrame);
 
        // accumulate mel frames
        std::transform(embedding.begin(), embedding.end(), melFrame.begin(), embedding.begin(), std::plus<float>());
 
        ++numFrames;
    }
 
    // average and normalise audio embedding
    numFrames = numFrames > 0 ? numFrames : 1;
    std::for_each(embedding.begin(), embedding.end(), [numFrames](float& v) { v = v / numFrames; });
    L2Normalise(embedding);
    assert(embedding.size() == 40);
 
    // apply autoencoder and renormalise
    embedding = applyAutoEncoder(embedding);
    L2Normalise(embedding);
    assert(embedding.size() == 20);
 
    return embedding;
}
 
void AudioEmbedding::L2Normalise(std::vector<float>& x, float eps)
{
    const auto norm = std::sqrt(std::inner_product(x.begin(), x.end(), x.begin(), eps));
    std::for_each(x.begin(), x.end(), [norm](float& v) { v = v / norm; });
}
 
std::vector<float> AudioEmbedding::applyMelFilterbank(const std::vector<float>& x)
{
    const auto n = 1 + m_fftSize / 2;
    std::vector<float> melFrame(m_mel.size(), 0.f);
    for (std::size_t i = 0; i < m_mel.size(); ++i)
    {
        melFrame[i] = std::inner_product(x.begin(), x.begin() + n, m_mel[i].begin(), 0.f);
    }
    return melFrame;
}
 
std::vector<float> AudioEmbedding::applyAutoEncoder(const std::vector<float>& x)
{
    std::vector<float> latent(m_weight.size(), 0.f);
    for (std::size_t i = 0; i < m_weight.size(); ++i)
    {
        latent[i] = std::inner_product(x.begin(), x.end(), m_weight[i].begin(), 0.f);
    }
    return latent;
}
 
std::vector<float> AudioEmbedding::hz_to_mel(std::vector<float> freqs, bool htk)
{
    std::vector<float> mels(freqs.size());
    if (htk)
    {
        for (std::size_t i = 0; i < mels.size(); ++i)
        {
            mels[i] = 2595.0f * std::log10f(1.0f + freqs[i] / 700.0f);
        }
        return mels;
    }
 
    const float fmin = 0.0f;
    const float f_sp = 200.0f / 3.0f;
 
    for (std::size_t i = 0; i < mels.size(); ++i)
    {
        mels[i] = (freqs[i] - fmin) / f_sp;
    }
 
    const float min_log_hz = 1000.0f;
    const float min_log_mel = (min_log_hz - fmin) / f_sp;
    const float logstep = std::log(6.4f) / 27.0f;
 
    for (std::size_t i = 0; i < mels.size(); ++i)
    {
        if (freqs[i] >= min_log_hz)
        {
            mels[i] = min_log_mel + std::log(freqs[i] / min_log_hz) / logstep;
        }
    }
 
    return mels;
}
 
std::vector<float> AudioEmbedding::mel_to_hz(std::vector<float> mels, bool htk)
{
    std::vector<float> freqs(mels.size());
    if (htk)
    {
        for (std::size_t i = 0; i < mels.size(); ++i)
        {
            freqs[i] = 700.0f * (std::pow(10.0f, mels[i] / 2595.0f) - 1.0f);
        }
        return freqs;
    }
 
    const float f_min = 0.0f;
    const float f_sp = 200.0f / 3.0f;
 
    for (std::size_t i = 0; i < mels.size(); ++i)
    {
        freqs[i] = f_min + f_sp * mels[i];
    }
 
    const float min_log_hz = 1000.0f;
    const float min_log_mel = (min_log_hz - f_min) / f_sp;
    const float logstep = std::log(6.4f) / 27.0f;
 
    for (std::size_t i = 0; i < mels.size(); ++i)
    {
        if (mels[i] >= min_log_mel)
        {
            freqs[i] = min_log_hz * std::exp(logstep * (mels[i] - min_log_mel));
        }
    }
 
    return freqs;
}
 
std::vector<float> AudioEmbedding::mel_frequencies(float fmin, float fmax, int n_mels, bool htk)
{
    const auto fmin_v = std::vector<float>(1, fmin);
    const auto fmax_v = std::vector<float>(1, fmax);
    const float min_mel = hz_to_mel(fmin_v, htk)[0];
    const float max_mel = hz_to_mel(fmax_v, htk)[0];
 
    const auto step = (max_mel - min_mel) / static_cast<float>(n_mels - 1);
    std::vector<float> mels = std::vector<float>(n_mels);
    for (int i = 0; i < n_mels; ++i)
    {
        mels[i] = min_mel + step * static_cast<float>(i);
    }
 
    return mel_to_hz(mels, htk);
}
 
std::vector<std::vector<float>> AudioEmbedding::mel(int sr, int n_fft, int n_mels, float fmin, float fmax, bool htk)
{
    const int length = 1 + n_fft / 2;
    if (fmax < 0.0f)
    {
        fmax = static_cast<float>(sr) / 2.0f;
    }
 
    std::vector<std::vector<float>> weights(n_mels, std::vector<float>(length));
 
    std::vector<float> fft_freqs(length);
    for (int i = 0; i < length; ++i)
    {
        fft_freqs[i] = static_cast<float>(sr) / static_cast<float>(n_fft) * static_cast<float>(i);
    }
 
    auto mel_f = mel_frequencies(fmin, fmax, n_mels + 2, htk);
 
    std::vector<float> fdiff(mel_f.size() - 1);
    for (std::size_t i = 0; i < fdiff.size(); ++i)
    {
        fdiff[i] = mel_f[i + 1] - mel_f[i];
    }
 
    std::vector<std::vector<float>> ramps(mel_f.size(), std::vector<float>(fft_freqs.size()));
    for (std::size_t i = 0; i < mel_f.size(); ++i)
    {
        for (std::size_t j = 0; j < fft_freqs.size(); ++j)
        {
            ramps[i][j] = mel_f[i] - fft_freqs[j];
        }
    }
 
    auto lower = std::vector<float>(fft_freqs.size());
    auto upper = std::vector<float>(fft_freqs.size());
    for (int i = 0; i < n_mels; ++i)
    {
        for (std::size_t j = 0; j < lower.size(); j++)
        {
            lower[j] = -1 * ramps[i][j] / fdiff[i];
        }
 
        for (std::size_t j = 0; j < lower.size(); ++j)
        {
            upper[j] = ramps[i + 2][j] / fdiff[i + 1];
        }
 
        for (std::size_t j = 0; j < lower.size(); ++j)
        {
            auto lower_upper_minimum = 0.0f;
            if (lower[j] > upper[j])
            {
                lower_upper_minimum = upper[j];
            }
            else
            {
                lower_upper_minimum = lower[j];
            }
 
            if (lower_upper_minimum > 0.0f)
            {
                weights[i][j] = lower_upper_minimum;
            }
            else
            {
                weights[i][j] = 0.0f;
            }
        }
    }
 
    for (int i = 0; i < n_mels; ++i)
    {
        const auto enorm = 2.0f / (mel_f[2 + i] - mel_f[i]);
        for (int j = 0; j < length; ++j)
        {
            weights[i][j] = enorm * weights[i][j];
        }
    }
 
    return weights;
}
 
void AudioEmbedding::writeMatrixToFile(const std::vector<std::vector<float>>& matrix, const std::string& filename)
{
    std::ofstream outputFile(filename);
    if (!outputFile.is_open())
    {
        DBG("Failed to open file: ");
        return;
    }
 
    for (const auto& row : matrix)
    {
        for (const auto& value : row)
        {
            outputFile << value << " ";
        }
        outputFile << '\n';
    }
    outputFile.close();
}
 
} // namespace krotos