KrotosAudioBufferDSP.h

Go to the documentation of this file.

/*
==============================================================================
 
KrotosAudioBufferDSP.h
 
Created:
    13 June 2019 12:22:00am S White
 
Contributers:
    Team Krotos
 
==============================================================================
*/
 
#pragma once
 
// #define WRITE_FILES
 
#include <juce_graphics/juce_graphics.h>
#include <krotos_ml/krotos_ml.h>
 
namespace krotos
{
 
    struct AnalysisCoefficientsPhase
    {
        float tracking;
        float cycles;
        float cal0;
        float cal3;
        float cal4;
        float cal5;
    };
 
    struct AnalysisCoefficients
    {
        float coeff0{0.f};
        float coeff1{0.f};
        float coeff2{0.f};
        float coeff3{0.f};
        float coeff4{0.f};
        float coeff5{0.f};
        float coeff6{0.f};
        float coeff7{0.f};
        float tracking{0.f};
        float selectX{0.f};
        float selectY{0.f};
        float selectZ{0.f};
        float dispAAlpha{0.f};
        float dispAH{0.f};
        float dispAA{0.f};
    };
 
    class ProgressTracker
    {
 
      public:
        String& getMessage() { return m_message; }
 
        float getFilledIndicator() { return m_proportionUsage; }
 
        float getIndicator() { return m_indicator; }
 
        void setNumStages(int numStages)
        {
            m_numStages = numStages;
            m_increment = 1.f / static_cast<float>(numStages);
            m_message = "";
            m_indicator = 0.f;
            m_proportionUsage = 0.f;
            m_isLoading = true;
        }
 
        void stage(String message)
        {
            m_message = message;
            m_indicator = m_proportionUsage;
            m_proportionUsage += m_increment;
            m_isLoading = true;
        }
 
        void end(String finalMessage)
        {
            m_message = finalMessage;
            m_indicator = 1.f;
            m_isLoading = false;
        }
 
        bool isLoading() { return m_isLoading; }
 
      private:
        int m_numStages{0};
        int m_currentStage{0};
 
        volatile bool m_isLoading{false};
 
        volatile float m_indicator = 0.f;
        float m_increment = 0.f;
        float m_proportionUsage = 0.f;
        String m_message{"---"};
    };
 
    class GrainRange
    {
 
      public:
        int clampToLimits(int val)
        {
            if (val > m_maxLimit)
                val = m_maxLimit;
            if (val < m_minLimit)
                val = m_minLimit;
            return val;
        }
 
        void set(int centre, float percent)
        {
            int halfSpan = int(float(centre) * percent);
            min = clampToLimits(centre - halfSpan);
            max = clampToLimits(centre + halfSpan);
        }
 
        void set(GrainRange range)
        {
            min = clampToLimits(range.min);
            max = clampToLimits(range.max);
        }
 
        void setLimits(GrainRange limits)
        {
            m_minLimit = limits.min;
            m_maxLimit = limits.max;
        }
 
        void setToLimits()
        {
            min = m_minLimit;
            max = m_maxLimit;
        }
 
        void setLimits(int newMin, int newMax)
        {
            m_minLimit = newMin;
            m_maxLimit = newMax;
        }
 
        bool isWithinRange(int val) { return ((val >= min) && (val <= max)); }
 
        int min;
        int max;
 
      private:
        int m_minLimit;
        int m_maxLimit;
    };
 
    struct FrequencyRange
    {
        float min;
        float max;
    };
 
    struct AudioDescriptor
    {
        float frequency{0.0f};        // Instantaneous frequency at this grain
        int grainIndexByFrequency{0}; // Index to the grain
        int audioIndex{0};            // Index into the PCM audio buffer
        float grainSize{0.0f};        // Number of samples in the grain
        int grainIndexByTime{0};      // Index to the grain
        float principalX{0.f};
        float principalY{0.f};
        float principalZ{0.f};
        float principalQ{0.f};
    };
 
    template <typename Functor>
    void normaliseAudioDescriptorFeature(std::vector<AudioDescriptor>& decriptors, Functor functor, float border = 0.f)
    {
        float low = border;
        float high = 1.f - border;
        float min = 10000000000.f, max = -10000000000.f;
        // Find the min and max
        for (auto& item : decriptors)
        {
            float* target = functor(item);
            if (*target < min)
                min = *target;
            if (*target > max)
                max = *target;
        }
        if (approximatelyEqual(min, max))
        {
            // Set values to a constant
            for (auto& item : decriptors)
            {
                float* target = functor(item);
                *target = 0.5;
            }
        }
        else
        {
            // Scale the values
            for (auto& item : decriptors)
            {
                float* target = functor(item);
                *target = jmap<float>(*target, min, max, low, high);
            }
        }
    }
 
    template <typename Functor2>
    void normaliseAndInvertAudioDescriptorFeature(std::vector<AudioDescriptor>& decriptors, Functor2 functor,
                                                  float border = 0.f)
    {
        float low = border;
        float high = 1.f - border;
        float min = 10000000000.f, max = -10000000000.f;
        // Find the min and max
        for (auto& item : decriptors)
        {
            float* target = functor(item);
            if (*target < min)
                min = *target;
            if (*target > max)
                max = *target;
        }
        if (approximatelyEqual(min, max))
        {
            // Set values to a constant
            for (auto& item : decriptors)
            {
                float* target = functor(item);
                *target = 0.5;
            }
        }
        else
        {
            // Scale the values
            for (auto& item : decriptors)
            {
                float* target = functor(item);
                *target = jmap<float>(*target, min, max, high, low);
            }
        }
    }
 
    class KrotosAudioBufferDSP : public KrotosAudioBuffer, public Thread, public Thread::Listener, public Timer
    {
      public:
        KrotosAudioBufferDSP();
 
        ~KrotosAudioBufferDSP()
        {
            // TODO: Test revealed that the thread was still running if
            // analysis is continuing when the Kwidget owner has been
            // destroyed. This fixes it for now at least...
            stopThread(ANALYSIS_TIMER_PERIOD);
        };
 
        void exitSignalSent() override{};
 
        // Note: these must match Kwidget_Reformer::SegmentationMethods
        enum class SegmentationMethod
        {
            SegmentationNone = 0,
            STFT = 32,     // Could be renamed with a global replace later on
            EnvOnset = 64, // etc ...
            OverlapWindow = 96,
            SuperFlux = 128,
            NUM_SEGMENTATIONS = 129
        };
 
        // Note: these must match Kwidget_Reformer::FeatureExtrMethods
        enum class FeatureExtrMethod
        {
            FeatureExtrMethodNone = 0,
            Harmonic_PitchEstimation = 1,
            MFCC_PCA = 32,       // Could be renamed with a global replace later on
            Centroid_Level = 64, // etc ...
            SpCentroid_RMS = 96,
            SpFlatness_RMS = 128,
            ERBSpCentroid_RMS = 160,
            ERBSpCentroid_ERBSpFlatness = 192,
            NUM_FEATUREMETHODS = 193
        };
 
        enum class AnalysisScheme
        {
            SchemeNone = 0,
            Vehicle,
            GeneralScheme,
            NUM_SCHEMES
        };
 
        void generalScheme();
 
        void writeSpectrogramToFile(const std::vector<std::vector<float>>& spectrogram, const juce::String filename)
        {
            juce::File outputFile = juce::File::getCurrentWorkingDirectory().getChildFile(filename);
            juce::FileOutputStream outputFileStream(outputFile);
            if (outputFileStream.openedOk())
            {
                // Write matrix elements to the file
                for (const auto& row : spectrogram)
                {
                    for (const auto& value : row)
                    {
                        outputFileStream << value << " ";
                    }
                    outputFileStream << '\n';
                }
                outputFileStream.flush();
            }
            else
            {
                DBG("Error writting");
            }
        }
 
        void writeFeatureToFile(const std::vector<float>& feature, const std::string& filename)
        {
            std::ofstream file(filename);
            if (file.is_open())
            {
                for (const float value : feature)
                {
                    file << std::to_string(value) << "\n";
                }
                file.close();
                DBG("Feature written to file: ");
            }
            else
            {
                DBG("Failed to open file: ");
            }
        }
 
        float crossCorrelate(const float* referencePointer, const float* candidatePointer, const int testSize);
 
        float autoCorrelateCentred(int centerIndex, int testSize);
 
        int calcFreeGrain(int centerIndex, int previousGrainSize, GrainRange range);
 
        void setPhaseAnalysisCoefficients(AnalysisCoefficientsPhase coeff) { m_phaseAnalysisCoefficients = coeff; }
 
        int size() { return getNumSamples(); }
 
        float* data() { return getWritePointer(0); }
 
        AnalysisCoefficients& getAnalysisCoefficients();
 
        void analyse();
 
        void analysePhase();
 
        void setDefaultPhaseAnalysisCoefficients();
 
        void invalidate() { m_trackingDataIsValid = false; }
 
        void validate()
        {
            m_trackingDataIsValid = true;
            triggerDisplayCacheRegeneration();
        }
 
        bool analysisResultsAreValid() { return m_trackingDataIsValid; }
 
        void triggerDisplayCacheRegeneration() { m_requestDisplayCacheRegeneration = true; }
 
        bool shouldRegenerateDisplayCache();
 
        bool scanForGrainsFreeMarker(std::vector<int>& returnGrains, const int audioIndexStart,
                                     const int useableIndexEnd, const int startGrainSize);
        bool scanBackwardsForGrainsFreeMarker(std::vector<int>& returnGrains, const int audioIndexStart,
                                              const int useableIndexEnd, const int startGrainSize);
 
        void buildDescriptors(std::vector<AudioDescriptor>& descriptors, const std::vector<int>& grainsList,
                              const int startGrainAudioIndex, bool goingBackwards);
 
        void setAnalysisWavRange(float start, float end)
        {
            m_startPos = start;
            m_endPos = end;
        }
 
        void setAnalysisGrainSizeRange();
 
        std::vector<AudioDescriptor>& getGrainDescriptionByTime() { return m_grainDescriptionByTime; }
 
        std::vector<AudioDescriptor>& getGrainDescriptionByZ() { return m_grainDescriptionByZ; }
 
        std::size_t getNearestNeighbour2D(float x, float y) { return m_nnSearch->knnQuery(x, y, 0.f); }
 
        std::vector<std::size_t> getNearestNeighbour2D(float x, float y, std::size_t k)
        {
            return m_nnSearch->knnQuery(x, y, 0.f, k);
        }
 
        std::vector<std::size_t> getNearestNeighbour2D(float x, float y, float search_radius)
        {
            return m_nnSearch->radiusSearch(x, y, 0.f, search_radius);
        }
 
        std::size_t getNearestNeighbour3D(float x, float y, float z) { return m_nnSearch->knnQuery(x, y, z); }
 
        std::vector<std::size_t> getNearestNeighbour3D(float x, float y, float z, std::size_t k)
        {
            return m_nnSearch->knnQuery(x, y, z, k);
        }
 
        std::vector<std::size_t> getNearestNeighbour3D(float x, float y, float z, float search_radius)
        {
            return m_nnSearch->radiusSearch(x, y, z, search_radius);
        }
 
        void setTrackingValue(float newValue) { m_trackingValue = newValue; }
 
        void setTrackingCycles(float newValue) { m_trackingCycles = newValue; }
 
        AudioDescriptor& audioIndexToDescriptor(int audioIndex);
        AudioDescriptor& audioPercentToDescriptor(float audioPercent);
        AudioDescriptor& audioFrequencyToDescriptor(float frequency);
 
        inline void clampGrainDescriptionByTimeIndex(size_t& index);
 
        inline StereoSample getSampleFromGrainPhase(double grainPhase);
 
        inline double getFrequencyFromGrainPhase(double grainPhase);
 
        ProgressTracker& getProgressTracker();
 
        SegmentationMethod getSegmentationMethod() const { return m_segmentationMethod; }
 
        FeatureExtrMethod getFeatureExtractionMethod() { return m_featureMethod; }
 
        AnalysisScheme getAnalysisScheme() { return m_analysisScheme; }
 
        void setSegmentationMethod(SegmentationMethod segmentation) { m_segmentationMethod = segmentation; }
 
        void setFeatureExtrMethod(FeatureExtrMethod featureExtraction) { m_featureMethod = featureExtraction; }
 
        void setAnalysisScheme(AnalysisScheme scheme) { m_analysisScheme = scheme; }
 
      private:
        const int ANALYSIS_TIMER_PERIOD{250};
        void timerCallback() override;
 
        void run() override;
 
        void cleanupAnalysis();
 
        std::vector<float> averageBufferToMono();
        std::vector<std::vector<float>> segmentOnsetSlices(std::vector<int> onsetPositions,
                                                           std::vector<float> signalBufferMono);
        std::vector<std::vector<float>> segmentChunks(std::vector<int> onsetPositions,
                                                      std::vector<float> signalBufferMono, std::size_t window_length);
 
        std::vector<int> chunk(const std::vector<float>& bufferVector, std::size_t window_length,
                               std::size_t hop_length);
        std::vector<int> envelopeOnsetDetection(const std::vector<float>& bufferVector, float samplerate);
        std::vector<int> superFluxOnsetDetection(float samplerate); // TODO See what happens here with
                                                                    // averaging to mono!!!!
 
        std::vector<AudioDescriptor> getDescriptorsTime(const std::vector<std::vector<float>>& timeDomainSlices,
                                                        const std::vector<int>& onsetPositions);
        std::vector<std::vector<float>> mfccFeatureExtraction(std::vector<std::vector<float>>& timeDomainSlices,
                                                              const std::vector<int>& onsetPositions);
        std::vector<std::vector<float>> temporalFeatureExtraction(std::vector<std::vector<float>>& timeDomainSlices);
        std::vector<std::vector<float>> specCntrRMS_FeatureExtraction(
                std::vector<std::vector<float>>& timeDomainSlices);
        std::vector<std::vector<float>> specFlatnessRMS_FeatureExtraction(
                std::vector<std::vector<float>>& timeDomainSlices);
        std::vector<std::vector<float>> erbSpecCentroidRMS_FeatureExtraction(
                std::vector<std::vector<float>>& timeDomainSlices);
        std::vector<std::vector<float>> erbSpecCentroidFlatness_FeatureExtraction(
                std::vector<std::vector<float>>& timeDomainSlices);
        std::vector<std::vector<float>> harmonicPitchEstimation(std::vector<std::vector<float>>& timeDomainSlices);
        // TODO:: Fix pitch tracking !!
        std::vector<std::vector<float>> pitchPeakRMSFeatureExtraction(
                std::vector<std::vector<float>>& timeDomainSlices);
 
        void applyPCA(std::vector<std::vector<float>>& features, const std::size_t& n_components);
        void setFeaturesAndNormalise(const std::vector<std::vector<float>>& features,
                                     std::vector<AudioDescriptor>& descriptorsTime);
        void buildKNNIndex(const std::vector<AudioDescriptor>& descriptorsTime);
        void buildKNNIndexWithZ(const std::vector<AudioDescriptor>& descriptorsTime);
        void sortByZ(const std::vector<AudioDescriptor>& descriptorsTime);
 
        SegmentationMethod m_segmentationMethod{SegmentationMethod::SegmentationNone}; // Default to none
        FeatureExtrMethod m_featureMethod{FeatureExtrMethod::FeatureExtrMethodNone};   // Default to
                                                                                       // none
        AnalysisScheme m_analysisScheme{AnalysisScheme::SchemeNone};                   // Default to none
        AnalysisCoefficients m_AnalysisCoefficients;
 
        ProgressTracker m_progressTracker;
 
        float m_trackingValue{0.5f};
        float m_trackingCycles{1.0f};
 
        GrainRange m_analysisGrainSizeRange;
        volatile std::atomic<bool> m_trackingDataIsValid{false};
        volatile std::atomic<bool> m_requestDisplayCacheRegeneration{false};
 
        // Vector containing grain boundary descriptors, sorted in frequency order
        std::vector<AudioDescriptor> m_grainDescriptionByFrequency;
 
        // Vector containing grain boundary descriptors, sorted by time (the order they appear in the source wav)
        std::vector<AudioDescriptor> m_grainDescriptionByTime;
 
        std::vector<AudioDescriptor> m_grainDescriptionByZ;
 
        std::unique_ptr<KDTree> m_nnSearch;
        std::unique_ptr<KDTree> m_grainMapper;
        std::unique_ptr<PCA> m_principalComponentAnalysis;
 
        float m_startPos{0.0f};
        float m_endPos{1.0f};
 
        AnalysisCoefficientsPhase m_phaseAnalysisCoefficients;
 
        // Segmentation results
        std::vector<std::vector<float>> m_timeDomainSlices;
        std::vector<int> m_onsetPositions;
        std::vector<AudioDescriptor> m_descriptorsTime;
    };
} // namespace krotos